CogSci 2020 Analysis

Packages and utilities

library(tidyverse)

Registered S3 method overwritten by 'dplyr':
  method           from
  print.rowwise_df     
Registered S3 methods overwritten by 'dbplyr':
  method         from
  print.tbl_lazy     
  print.tbl_sql      
── Attaching packages ───────────────────────────────────────────────────────────── tidyverse 1.3.0 ──
✓ ggplot2 3.2.1     ✓ purrr   0.3.3
✓ tibble  2.1.3     ✓ dplyr   0.8.4
✓ tidyr   1.0.2     ✓ stringr 1.4.0
✓ readr   1.3.1     ✓ forcats 0.4.0
── Conflicts ──────────────────────────────────────────────────────────────── tidyverse_conflicts() ──
x dplyr::filter() masks stats::filter()
x dplyr::lag()    masks stats::lag()

library(lme4)

Loading required package: Matrix

Attaching package: ‘Matrix’

The following objects are masked from ‘package:tidyr’:

    expand, pack, unpack

library(lmerTest)

Attaching package: ‘lmerTest’

The following object is masked from ‘package:lme4’:

    lmer

The following object is masked from ‘package:stats’:

    step

library(logging)
library(mvtnorm)
library(mgcv)

Loading required package: nlme

Attaching package: ‘nlme’

The following object is masked from ‘package:lme4’:

    lmList

The following object is masked from ‘package:dplyr’:

    collapse

This is mgcv 1.8-31. For overview type 'help("mgcv-package")'.

# Provides bootstrap resampling tools
library(rsample)

# Compute the log-likelihood of a new dataset using a fit lme4 model.
logLik_test <- function(lm, test_X, test_y) {
  predictions <- predict(lm, test_X, re.form=NA)
  # Get std.dev. of residual, estimated from train data
  stdev <- sigma(lm)
  # For each prediction--observation, get the density p(obs | N(predicted, model_sigma)) and reduce
  density <- sum(dnorm(test_y, predictions, stdev, log=TRUE))
  return(density)
}
# Get per-prediction log-likelihood
logLik_test_per <- function(lm, test_X, test_y) {
  predictions <- predict(lm, test_X, re.form=NA)
  # Get std.dev. of residual, estimated from train data
  stdev <- sigma(lm)
  # For each prediction--observation, get the density p(obs | N(predicted, model_sigma))
  densities <- dnorm(test_y, predictions, stdev, log=TRUE)
  return(densities)
}
# Compute MSE of a new dataset using a fit lme4 model.
mse_test <- function(lm, test_X, test_y) {
  return(mean((predict(lm, test_X, re.form=NA) - test_y) ^ 2))
}
#Sanity checks
#mylm <- gam(psychometric ~  s(surprisal, bs = "cr", k = 20) + s(prev_surp, bs = "cr", k = 20) + te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr"), data=train_data)
#c(logLik(mylm), logLik_test(mylm, train_data, train_data$psychometric))
#logLik_test(mylm, test_data, test_data$psychometric)

Data loading and preprocessing

data = read.csv("../data/harmonized_results.csv")

all_data = data %>%
  mutate(seed = as.factor(seed)) %>%
  group_by(corpus, model, training, seed) %>%
    mutate(prev_surp = lag(surprisal),
         prev_code = lag(code),
         prev_len = lag(len),
         prev_freq = lag(freq),
         prev_surp = lag(surprisal),
         
         prev2_freq = lag(prev_freq),
         prev2_code = lag(prev_code),
         prev2_len = lag(prev_len),
         prev2_surp = lag(prev_surp),
         
         prev3_freq = lag(prev2_freq),
         prev3_code = lag(prev2_code),
         prev3_len = lag(prev2_len),
         prev3_surp = lag(prev2_surp),
         
         prev4_freq = lag(prev3_freq),
         prev4_code = lag(prev3_code),
         prev4_len = lag(prev3_len),
         prev4_surp = lag(prev3_surp)) %>%
  ungroup() %>%
  
  # Filter back two for the dundee corpus. Filter back 1 for all other corpora
  # NB this effectively removes all zero-surprisal rows, since early-sentence tokens don't have contiguous token history
  filter((corpus == "dundee" & code == prev2_code + 2) | (corpus != "dundee" & code == prev4_code + 4)) %>%
  
  select(-prev_code, -prev2_code, -prev3_code) %>%
  drop_na()

all_data = all_data %>%
  mutate(
    model = as.character(model),
    model = if_else(model == "gpt-2", "gpt2", model),
    model = as.factor(model))

missing_rows = all_data %>% complete(nesting(corpus, code), nesting(model, training, seed)) %>% 
  group_by(corpus, code) %>% 
    filter(sum(is.na(surprisal)) > 0) %>% 
  ungroup() %>% 
  anti_join(all_data, by=c("corpus", "code", "model", "training", "seed"))

missing_rows %>% ggplot(aes(x=corpus, fill=factor(paste(model,training)))) +
geom_bar(position=position_dodge(width=0.8))

print(missing_rows %>% group_by(model, training, seed, corpus) %>% summarise(n=n()) %>% arrange(desc(n)))

# Compute the ideal number of model--seed--training observations per token.
to_drop = all_data %>%
  group_by(corpus, code) %>% summarise(n = n()) %>% ungroup() %>%
  group_by(corpus) %>% mutate( max_n = max(n)) %>% ungroup() %>%
  filter(max_n != n) %>% 
  select(code, corpus)

# Find tokens which have 28 observations and compare model+training freqs
all_data %>% filter(corpus == "bnc-brown") %>% group_by(code) %>% filter(n() == 28)

tempx = all_data %>% filter(corpus == "bnc-brown", code == 17103)
table(paste(tempx$model, tempx$training))

      5gram bllip-lg       5gram bllip-md       5gram bllip-sm       5gram bllip-xs        gpt2 bllip-lg gpt2 bllip-lg-gptbpe        gpt2 bllip-md gpt2 bllip-md-gptbpe gpt2 bllip-sm-gptbpe gpt2 bllip-xs-gptbpe        rnng bllip-lg 
                   1                    1                    1                    1                    1                    1                    1                    1                    1                    1                    1 
       rnng bllip-md        rnng bllip-sm        rnng bllip-xs     vanilla bllip-lg     vanilla bllip-md     vanilla bllip-sm     vanilla bllip-xs 
                   2                    2                    4                    1                    3                    3                    3 

# This one is missing a seed
#to_drop %>% filter(corpus == "bnc-brown") %>% arrange(code)
tempx = all_data %>% filter(corpus == "bnc-brown", code == 17017)
table(paste(tempx$model, tempx$training))

      5gram bllip-lg       5gram bllip-md       5gram bllip-sm       5gram bllip-xs        gpt2 bllip-lg gpt2 bllip-lg-gptbpe        gpt2 bllip-md gpt2 bllip-md-gptbpe gpt2 bllip-sm-gptbpe gpt2 bllip-xs-gptbpe        rnng bllip-lg 
                   1                    1                    1                    1                    1                    1                    1                    1                    1                    1                    1 
       rnng bllip-md        rnng bllip-sm        rnng bllip-xs     vanilla bllip-lg     vanilla bllip-md     vanilla bllip-sm     vanilla bllip-xs 
                   2                    2                    4                    1                    3                    3                    3 

# nvm somehow not a problem anymore ..
# # Zooming in on the problem -- why is there no bllip-lg data here?
# all_data %>% filter(corpus == "dundee", model == "vanilla", training == "bllip-lg", code > 10720, code < 10730)

loginfo(paste("Dropping", nrow(to_drop), "observations corresponding to corpus tokens which are missing observations for some model."))

2021-04-03 12:20:19 INFO::Dropping 10561 observations corresponding to corpus tokens which are missing observations for some model.

loginfo(paste("Dropping", to_drop %>% group_by(corpus, code) %>% n_groups(), "tokens which are missing observations for some model."))

2021-04-03 12:20:19 INFO::Dropping 10561 tokens which are missing observations for some model.

all_data = all_data %>% anti_join(to_drop %>% group_by(corpus, code), by=c("corpus", "code"))
loginfo(paste("After drop,", nrow(all_data), "observations (", all_data %>% group_by(corpus, code) %>% n_groups(), " tokens) remain."))

2021-04-03 12:20:19 INFO::After drop, 960335 observations ( 33115  tokens) remain.

to_drop_zero_surps = all_data %>% group_by(corpus, code) %>% filter(any(surprisal == 0)) %>% ungroup()
loginfo(paste("Dropping", nrow(to_drop_zero_surps), "observations corresponding to corpus tokens which have surprisal zeros for some model."))

2021-04-03 12:20:25 INFO::Dropping 116 observations corresponding to corpus tokens which have surprisal zeros for some model.

loginfo(paste("Dropping", to_drop_zero_surps %>% group_by(corpus, code) %>% n_groups(), "tokens which have surprisal zeros for some model."))

2021-04-03 12:20:26 INFO::Dropping 4 tokens which have surprisal zeros for some model.

all_data = all_data %>% anti_join(to_drop_zero_surps %>% group_by(corpus, code), by=c("corpus", "code"))
loginfo(paste("After drop,", nrow(all_data), "observations (", all_data %>% group_by(corpus, code) %>% n_groups(), " tokens) remain."))

2021-04-03 12:20:26 INFO::After drop, 960219 observations ( 33111  tokens) remain.

to_drop_zero_psychs = all_data %>% group_by(corpus, code) %>% filter(any(psychometric == 0)) %>% ungroup()
loginfo(paste("Dropping", nrow(to_drop_zero_psychs), "observations corresponding to corpus tokens which have psychometric zeros for some model."))

2021-04-03 12:20:30 INFO::Dropping 14935 observations corresponding to corpus tokens which have psychometric zeros for some model.

loginfo(paste("Dropping", to_drop_zero_psychs %>% group_by(corpus, code) %>% n_groups(), "tokens which have psychometric zeros for some model."))

2021-04-03 12:20:30 INFO::Dropping 515 tokens which have psychometric zeros for some model.

all_data = all_data %>% anti_join(to_drop_zero_psychs %>% group_by(corpus, code), by=c("corpus", "code"))
loginfo(paste("After drop,", nrow(all_data), "observations (", all_data %>% group_by(corpus, code) %>% n_groups(), " tokens) remain."))

2021-04-03 12:20:30 INFO::After drop, 945284 observations ( 32596  tokens) remain.

Train Linear Models which are used to assess Delta Log Lik

# Compute linear model stats for the given training data subset and full test data.
# Automatically subsets the test data to match the relevant group for which we are training a linear model.
get_lm_data <- function(df, test_data, formula, fold, store_env) {
  #this_lm <- gam(formula, data=df);
  print(paste(unique(paste(df$model, df$training, df$seed, df$corpus))[1], fold, nrow(df)))
  this_lm = lm(formula, data=df)
  this_test_data <- semi_join(test_data, df, by=c("training", "model", "seed", "corpus"));
  
  # Save lm to the global env so that we can access residuals later.
  lm_name = paste(unique(paste(df$model, df$training, df$seed, df$corpus))[1], fold)
  assign(lm_name, this_lm, envir=store_env)
  
  summarise(df,
            log_lik = as.numeric(logLik(this_lm, REML = F)),
            test_lik = logLik_test(this_lm, this_test_data, this_test_data$psychometric),
            test_mse = mse_test(this_lm, this_test_data, this_test_data$psychometric))
}
# For a previously fitted lm stored in store_env, get the residuals on test data of the relevant data subset.
get_lm_residuals <- function(df, fold, store_env) {
  # Retrieve the relevant lm.
  lm_name = paste(unique(paste(df$model, df$training, df$seed, df$corpus))[1], fold)
  this_lm <- get(lm_name, envir=store_env)
  
  mutate(df,
         likelihood = logLik_test_per(this_lm, df, df$psychometric),
         resid = df$psychometric - predict(this_lm, df, re.form=NA))
}
# Compute per-example delta-log-likelihood for the given test fold.
get_lm_delta_log_lik <- function(test_data, fold, baseline_env, full_env) {
  lm_name = paste(unique(paste(test_data$model, test_data$training, test_data$seed, test_data$corpus))[1], fold)
  baseline_lm <- get(lm_name, envir=baseline_env)
  full_lm <- get(lm_name, envir=full_env)
  
  delta_log_lik = logLik_test_per(full_lm, test_data, test_data$psychometric) - logLik_test_per(baseline_lm, test_data, test_data$psychometric)
  return(cbind(test_data, delta_log_lik=delta_log_lik))
}
#####
# Define regression formulae.

# Regression code to fit GAM models.
#baseline_rt_regression = psychometric ~ te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr") + te(prev2_freq, prev2_len, bs = "cr")
#baselie_sprt_regression = psychometric ~ te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr") + te(prev2_freq, prev2_len, bs = "cr") + te(prev3_freq, prev3_len, bs = "cr") + te(prev4_freq, prev4_len, bs = "cr")

#full_rt_regression = psychometric ~ s(surprisal, bs = "cr", k = 20) + s(prev_surp, bs = "cr", k = 20) + s(prev2_surp, bs = "cr", k = 20) + te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr") + te(prev2_freq, prev2_len, bs = "cr")
#full_sprt_regression = psychometric ~ s(surprisal, bs = "cr", k = 20) + s(prev_surp, bs = "cr", k = 20) + s(prev2_surp, bs = "cr", k = 20) + s(prev3_surp, bs = "cr", k = 20) + s(prev4_surp, bs = "cr", k = 20) + te(freq, len, bs = "cr") + te(prev_freq, prev_len, bs = "cr") + te(prev2_freq, prev2_len, bs = "cr") + te(prev3_freq, prev3_len, bs = "cr") + te(prev4_freq, prev4_len, bs = "cr")

# Regression Code to fit linear models
baseline_rt_regression = psychometric ~ freq + prev_freq + prev2_freq + len + prev_len + prev2_len
baseline_sprt_regression = psychometric ~ freq + prev_freq + prev2_freq + prev3_freq + prev4_freq + len + prev_len + prev2_len + prev3_len + prev4_len

full_sprt_regression = psychometric ~ surprisal + prev_surp + prev2_surp + prev3_surp + prev4_surp + freq + prev_freq + prev2_freq + prev3_freq + prev4_freq + len + prev_len + prev2_len + prev3_len + prev4_len
full_rt_regression = psychometric ~ surprisal + prev_surp + prev2_surp + freq + prev_freq + prev2_freq + len + prev_len + prev2_len
  
#####
# Prepare frames/environments for storing results/objects.
baseline_results = data.frame()
full_model_results = data.frame()
baseline_residuals = data.frame()
full_residuals = data.frame()
log_lik_deltas = data.frame()

#Randomly shuffle the data
all_data<-all_data[sample(nrow(all_data)),]
#Create K equally size folds
K = 10
folds <- cut(seq(1,nrow(all_data)),breaks=K,labels=FALSE)
#Perform 10 fold cross validation

# Fit models for some fold of the data.
baseline_corpus = function(corpus, df, test_data, fold, env) {
  if(corpus == "dundee") {
    get_lm_data(df, test_data, baseline_rt_regression, fold, env)
  } else {
    get_lm_data(df, test_data, baseline_sprt_regression, fold, env)
  }
}
full_model_corpus = function(corpus, df, test_data, fold, env) {
  if(corpus[1] == "dundee") {
    get_lm_data(df, test_data, full_rt_regression, fold, env)
  } else {
    get_lm_data(df, test_data, full_sprt_regression, fold, env)
  }
}

# Prepare a new Environment in which we store fitted LMs, which we'll query later for residuals and other metrics.
baseline_env = new.env()
full_env = new.env()

for(i in 1:K) { 
  #Segement your data by fold using the which() function 
  testIndexes <- which(folds==i, arr.ind=TRUE)
  test_data <- all_data[testIndexes, ]
  train_data <- all_data[-testIndexes, ]
  
  # Compute a baseline linear model for each model--training--seed--RT-corpus combination.
  baselines = train_data %>%
    group_by(model, training, seed, corpus) %>%
      print(model) %>%
      do(baseline_corpus(unique(.$corpus), ., test_data, i, baseline_env)) %>%
    ungroup() %>%
    mutate(seed = as.factor(seed),
           fold = i)
  
  baseline_results = rbind(baseline_results, baselines)
  
  # Compute a full linear model for each model--training--seed-RT-corpus combination
  full_models = train_data %>%
    group_by(model, training, seed, corpus) %>%
      do(full_model_corpus(unique(.$corpus), ., test_data, i, full_env)) %>%
    ungroup() %>%
    mutate(seed = as.factor(seed),
           fold = i)
  
  full_model_results = rbind(full_model_results, full_models)
  
  # Compute delta-log-likelihoods
  fold_log_lik_deltas = test_data %>%
    group_by(model, training, seed, corpus) %>%
      do(get_lm_delta_log_lik(., i, baseline_env, full_env)) %>%
    ungroup()

  log_lik_deltas = rbind(log_lik_deltas, fold_log_lik_deltas)
  
  fold_baseline_residuals = test_data %>%
    group_by(model, training, seed, corpus) %>%
      do(get_lm_residuals(., i, baseline_env)) %>%
    ungroup()

  baseline_residuals = rbind(baseline_residuals, fold_baseline_residuals)

  fold_full_residuals = test_data %>%
    group_by(model, training, seed, corpus) %>%
      do(get_lm_residuals(., i, full_env)) %>%
    ungroup()

  full_residuals = rbind(full_residuals, fold_full_residuals)
}

[1] "5gram bllip-lg 1111 bnc-brown 1 1686"
[1] "5gram bllip-lg 1111 dundee 1 24474"
[1] "5gram bllip-lg 1111 natural-stories 1 3103"
[1] "5gram bllip-md 1111 bnc-brown 1 1687"
[1] "5gram bllip-md 1111 dundee 1 24574"
[1] "5gram bllip-md 1111 natural-stories 1 3060"
[1] "5gram bllip-sm 1111 bnc-brown 1 1693"
[1] "5gram bllip-sm 1111 dundee 1 24566"
[1] "5gram bllip-sm 1111 natural-stories 1 3115"
[1] "5gram bllip-xs 1111 bnc-brown 1 1668"
[1] "5gram bllip-xs 1111 dundee 1 24505"
[1] "5gram bllip-xs 1111 natural-stories 1 3100"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 1 1696"
[1] "gpt2 bllip-lg 1587139950 dundee 1 24497"
[1] "gpt2 bllip-lg 1587139950 natural-stories 1 3098"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 1 1684"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 1 24598"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 1 3094"
[1] "gpt2 bllip-md 1586986276 bnc-brown 1 1694"
[1] "gpt2 bllip-md 1586986276 dundee 1 24505"
[1] "gpt2 bllip-md 1586986276 natural-stories 1 3078"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 1 24647"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 1 3058"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 1 1710"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 1 24633"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 1 3094"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 1 1684"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 1 24544"
[1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 1 3099"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 1 1698"
[1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 1 24578"

|=============================================================================                                                                                                                                             | 36% ~4 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 1 3119"

|================================================================================                                                                                                                                          | 37% ~4 s remaining     [1] "rnng bllip-lg 7245 bnc-brown 1 1705"

|==================================================================================                                                                                                                                        | 38% ~3 s remaining     [1] "rnng bllip-lg 7245 dundee 1 24532"

|=====================================================================================                                                                                                                                     | 39% ~3 s remaining     [1] "rnng bllip-lg 7245 natural-stories 1 3082"

|=======================================================================================                                                                                                                                   | 40% ~3 s remaining     [1] "rnng bllip-md 3602 bnc-brown 1 1685"

|==========================================================================================                                                                                                                                | 41% ~3 s remaining     [1] "rnng bllip-md 3602 dundee 1 24528"

|============================================================================================                                                                                                                              | 43% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 1 3111"

|===============================================================================================                                                                                                                           | 44% ~3 s remaining     [1] "rnng bllip-md 44862 bnc-brown 1 1673"
[1] "rnng bllip-md 44862 dundee 1 24469"

|====================================================================================================                                                                                                                      | 46% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 1 3096"

|======================================================================================================                                                                                                                    | 47% ~3 s remaining     [1] "rnng bllip-sm 7877 bnc-brown 1 1699"
[1] "rnng bllip-sm 7877 dundee 1 24472"

|===========================================================================================================                                                                                                               | 49% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 1 3115"

|==============================================================================================================                                                                                                            | 51% ~3 s remaining     [1] "rnng bllip-sm 64924 bnc-brown 1 1711"
[1] "rnng bllip-sm 64924 dundee 1 24591"

|===================================================================================================================                                                                                                       | 53% ~3 s remaining     [1] "rnng bllip-sm 64924 natural-stories 1 3112"

|=====================================================================================================================                                                                                                     | 54% ~3 s remaining     [1] "rnng bllip-xs 4301 bnc-brown 1 1670"

|========================================================================================================================                                                                                                  | 55% ~3 s remaining     [1] "rnng bllip-xs 4301 dundee 1 24461"

|==========================================================================================================================                                                                                                | 56% ~3 s remaining     [1] "rnng bllip-xs 4301 natural-stories 1 3069"

|=============================================================================================================================                                                                                             | 57% ~2 s remaining     [1] "rnng bllip-xs 28066 bnc-brown 1 1702"

|===============================================================================================================================                                                                                           | 59% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 1 24564"

|==================================================================================================================================                                                                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 1 3113"

|====================================================================================================================================                                                                                      | 61% ~2 s remaining     [1] "rnng bllip-xs 28068 bnc-brown 1 1708"

|=======================================================================================================================================                                                                                   | 62% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 1 24624"

|=========================================================================================================================================                                                                                 | 63% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 1 3091"

|============================================================================================================================================                                                                              | 64% ~2 s remaining     [1] "rnng bllip-xs 51272 bnc-brown 1 1705"

|==============================================================================================================================================                                                                            | 66% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 1 24502"

|=================================================================================================================================================                                                                         | 67% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 1 3107"

|===================================================================================================================================================                                                                       | 68% ~2 s remaining     [1] "vanilla bllip-lg 111 bnc-brown 1 1692"

|======================================================================================================================================================                                                                    | 69% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 1 24620"

|========================================================================================================================================================                                                                  | 70% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 1 3090"

|===========================================================================================================================================================                                                               | 71% ~2 s remaining     [1] "vanilla bllip-md 120 bnc-brown 1 1683"
[1] "vanilla bllip-md 120 dundee 1 24602"

|================================================================================================================================================================                                                          | 74% ~2 s remaining     [1] "vanilla bllip-md 120 natural-stories 1 3100"

|==================================================================================================================================================================                                                        | 75% ~2 s remaining     [1] "vanilla bllip-md 607 bnc-brown 1 1696"

|=====================================================================================================================================================================                                                     | 76% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 1 1712"

|=======================================================================================================================================================================                                                   | 77% ~1 s remaining     [1] "vanilla bllip-md 922 dundee 1 24512"

|==========================================================================================================================================================================                                                | 78% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 1 3054"

|============================================================================================================================================================================                                              | 79% ~1 s remaining     [1] "vanilla bllip-sm 111 bnc-brown 1 1697"

|===============================================================================================================================================================================                                           | 80% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 1 24424"

|=================================================================================================================================================================================                                         | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 1 3100"

|====================================================================================================================================================================================                                      | 83% ~1 s remaining     [1] "vanilla bllip-sm 120 bnc-brown 1 1702"
[1] "vanilla bllip-sm 120 dundee 1 24483"

|=========================================================================================================================================================================================                                 | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 1 3110"

|===========================================================================================================================================================================================                               | 86% ~1 s remaining     [1] "vanilla bllip-sm 922 bnc-brown 1 1700"
[1] "vanilla bllip-sm 922 dundee 1 24635"

|================================================================================================================================================================================================                          | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 1 3125"

|===================================================================================================================================================================================================                       | 90% ~1 s remaining     [1] "vanilla bllip-xs 111 bnc-brown 1 1694"
[1] "vanilla bllip-xs 111 dundee 1 24535"

|========================================================================================================================================================================================================                  | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 1 3073"

|==========================================================================================================================================================================================================                | 93% ~0 s remaining     [1] "vanilla bllip-xs 120 bnc-brown 1 1715"
[1] "vanilla bllip-xs 120 dundee 1 24563"

|===============================================================================================================================================================================================================           | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 1 3085"

|==================================================================================================================================================================================================================        | 97% ~0 s remaining     [1] "vanilla bllip-xs 922 bnc-brown 1 1699"

|====================================================================================================================================================================================================================      | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 1 24614"

|=======================================================================================================================================================================================================================   | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 1 3104"

|==========================================================================================================================================================================================================================|100% ~0 s remaining     [1] "5gram bllip-lg 1111 bnc-brown 1 1686"
[1] "5gram bllip-lg 1111 dundee 1 24474"
[1] "5gram bllip-lg 1111 natural-stories 1 3103"
[1] "5gram bllip-md 1111 bnc-brown 1 1687"
[1] "5gram bllip-md 1111 dundee 1 24574"
[1] "5gram bllip-md 1111 natural-stories 1 3060"
[1] "5gram bllip-sm 1111 bnc-brown 1 1693"
[1] "5gram bllip-sm 1111 dundee 1 24566"
[1] "5gram bllip-sm 1111 natural-stories 1 3115"
[1] "5gram bllip-xs 1111 bnc-brown 1 1668"
[1] "5gram bllip-xs 1111 dundee 1 24505"
[1] "5gram bllip-xs 1111 natural-stories 1 3100"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 1 1696"
[1] "gpt2 bllip-lg 1587139950 dundee 1 24497"
[1] "gpt2 bllip-lg 1587139950 natural-stories 1 3098"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 1 1684"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 1 24598"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 1 3094"
[1] "gpt2 bllip-md 1586986276 bnc-brown 1 1694"
[1] "gpt2 bllip-md 1586986276 dundee 1 24505"
[1] "gpt2 bllip-md 1586986276 natural-stories 1 3078"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 1 24647"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 1 3058"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 1 1710"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 1 24633"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 1 3094"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 1 1684"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 1 24544"
[1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 1 3099"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 1 1698"
[1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 1 24578"
[1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 1 3119"
[1] "rnng bllip-lg 7245 bnc-brown 1 1705"
[1] "rnng bllip-lg 7245 dundee 1 24532"
[1] "rnng bllip-lg 7245 natural-stories 1 3082"
[1] "rnng bllip-md 3602 bnc-brown 1 1685"
[1] "rnng bllip-md 3602 dundee 1 24528"
[1] "rnng bllip-md 3602 natural-stories 1 3111"
[1] "rnng bllip-md 44862 bnc-brown 1 1673"
[1] "rnng bllip-md 44862 dundee 1 24469"

|====================================================================================================                                                                                                                      | 46% ~2 s remaining     [1] "rnng bllip-md 44862 natural-stories 1 3096"

|======================================================================================================                                                                                                                    | 47% ~2 s remaining     [1] "rnng bllip-sm 7877 bnc-brown 1 1699"

|=========================================================================================================                                                                                                                 | 48% ~2 s remaining     [1] "rnng bllip-sm 7877 dundee 1 24472"

|===========================================================================================================                                                                                                               | 49% ~2 s remaining     [1] "rnng bllip-sm 7877 natural-stories 1 3115"

|==============================================================================================================                                                                                                            | 51% ~2 s remaining     [1] "rnng bllip-sm 64924 bnc-brown 1 1711"

|================================================================================================================                                                                                                          | 52% ~2 s remaining     [1] "rnng bllip-sm 64924 dundee 1 24591"

|===================================================================================================================                                                                                                       | 53% ~2 s remaining     [1] "rnng bllip-sm 64924 natural-stories 1 3112"

|=====================================================================================================================                                                                                                     | 54% ~2 s remaining     [1] "rnng bllip-xs 4301 bnc-brown 1 1670"

|========================================================================================================================                                                                                                  | 55% ~2 s remaining     [1] "rnng bllip-xs 4301 dundee 1 24461"

|==========================================================================================================================                                                                                                | 56% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 1 3069"

|=============================================================================================================================                                                                                             | 57% ~2 s remaining     [1] "rnng bllip-xs 28066 bnc-brown 1 1702"

|===============================================================================================================================                                                                                           | 59% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 1 24564"

|==================================================================================================================================                                                                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 1 3113"

|====================================================================================================================================                                                                                      | 61% ~2 s remaining     [1] "rnng bllip-xs 28068 bnc-brown 1 1708"
[1] "rnng bllip-xs 28068 dundee 1 24624"

|=========================================================================================================================================                                                                                 | 63% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 1 3091"

|============================================================================================================================================                                                                              | 64% ~2 s remaining     [1] "rnng bllip-xs 51272 bnc-brown 1 1705"

|==============================================================================================================================================                                                                            | 66% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 1 24502"

|=================================================================================================================================================                                                                         | 67% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 1 3107"

|===================================================================================================================================================                                                                       | 68% ~2 s remaining     [1] "vanilla bllip-lg 111 bnc-brown 1 1692"

|======================================================================================================================================================                                                                    | 69% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 1 24620"

|========================================================================================================================================================                                                                  | 70% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 1 3090"

|===========================================================================================================================================================                                                               | 71% ~2 s remaining     [1] "vanilla bllip-md 120 bnc-brown 1 1683"

|=============================================================================================================================================================                                                             | 72% ~1 s remaining     [1] "vanilla bllip-md 120 dundee 1 24602"

|================================================================================================================================================================                                                          | 74% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 1 3100"

|==================================================================================================================================================================                                                        | 75% ~1 s remaining     [1] "vanilla bllip-md 607 bnc-brown 1 1696"

|=====================================================================================================================================================================                                                     | 76% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 1 1712"

|=======================================================================================================================================================================                                                   | 77% ~1 s remaining     [1] "vanilla bllip-md 922 dundee 1 24512"

|==========================================================================================================================================================================                                                | 78% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 1 3054"

|============================================================================================================================================================================                                              | 79% ~1 s remaining     [1] "vanilla bllip-sm 111 bnc-brown 1 1697"
[1] "vanilla bllip-sm 111 dundee 1 24424"

|=================================================================================================================================================================================                                         | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 1 3100"

|====================================================================================================================================================================================                                      | 83% ~1 s remaining     [1] "vanilla bllip-sm 120 bnc-brown 1 1702"

|======================================================================================================================================================================================                                    | 84% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 1 24483"

|=========================================================================================================================================================================================                                 | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 1 3110"

|===========================================================================================================================================================================================                               | 86% ~1 s remaining     [1] "vanilla bllip-sm 922 bnc-brown 1 1700"
[1] "vanilla bllip-sm 922 dundee 1 24635"

|================================================================================================================================================================================================                          | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 1 3125"

|===================================================================================================================================================================================================                       | 90% ~1 s remaining     [1] "vanilla bllip-xs 111 bnc-brown 1 1694"

|=====================================================================================================================================================================================================                     | 91% ~1 s remaining     [1] "vanilla bllip-xs 111 dundee 1 24535"

|========================================================================================================================================================================================================                  | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 1 3073"

|==========================================================================================================================================================================================================                | 93% ~0 s remaining     [1] "vanilla bllip-xs 120 bnc-brown 1 1715"

|=============================================================================================================================================================================================================             | 94% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 1 24563"

|===============================================================================================================================================================================================================           | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 1 3085"

|==================================================================================================================================================================================================================        | 97% ~0 s remaining     [1] "vanilla bllip-xs 922 bnc-brown 1 1699"

|====================================================================================================================================================================================================================      | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 1 24614"

|=======================================================================================================================================================================================================================   | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 1 3104"

|==========================================================================================================================================================================================================================|100% ~0 s remaining     

[1] "5gram bllip-lg 1111 bnc-brown 2 1694"
[1] "5gram bllip-lg 1111 dundee 2 24547"
[1] "5gram bllip-lg 1111 natural-stories 2 3079"
[1] "5gram bllip-md 1111 bnc-brown 2 1688"
[1] "5gram bllip-md 1111 dundee 2 24574"
[1] "5gram bllip-md 1111 natural-stories 2 3121"
[1] "5gram bllip-sm 1111 bnc-brown 2 1685"
[1] "5gram bllip-sm 1111 dundee 2 24612"
[1] "5gram bllip-sm 1111 natural-stories 2 3106"
[1] "5gram bllip-xs 1111 bnc-brown 2 1675"
[1] "5gram bllip-xs 1111 dundee 2 24556"
[1] "5gram bllip-xs 1111 natural-stories 2 3108"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 2 1684"
[1] "gpt2 bllip-lg 1587139950 dundee 2 24520"
[1] "gpt2 bllip-lg 1587139950 natural-stories 2 3071"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 2 1696"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 2 24634"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 2 3096"
[1] "gpt2 bllip-md 1586986276 bnc-brown 2 1684"
[1] "gpt2 bllip-md 1586986276 dundee 2 24578"
[1] "gpt2 bllip-md 1586986276 natural-stories 2 3129"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 2 24544"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 2 3094"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 2 1720"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 2 24604"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 2 3105"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 2 1686"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 2 24507"
[1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 2 3081"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 2 1693"
[1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 2 24529"
[1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 2 3085"
[1] "rnng bllip-lg 7245 bnc-brown 2 1684"
[1] "rnng bllip-lg 7245 dundee 2 24464"
[1] "rnng bllip-lg 7245 natural-stories 2 3082"
[1] "rnng bllip-md 3602 bnc-brown 2 1700"
[1] "rnng bllip-md 3602 dundee 2 24518"
[1] "rnng bllip-md 3602 natural-stories 2 3104"
[1] "rnng bllip-md 44862 bnc-brown 2 1692"

|=================================================================================================                                                                                                                         | 45% ~2 s remaining     [1] "rnng bllip-md 44862 dundee 2 24513"

|====================================================================================================                                                                                                                      | 46% ~2 s remaining     [1] "rnng bllip-md 44862 natural-stories 2 3093"

|======================================================================================================                                                                                                                    | 47% ~2 s remaining     [1] "rnng bllip-sm 7877 bnc-brown 2 1707"

|=========================================================================================================                                                                                                                 | 48% ~2 s remaining     [1] "rnng bllip-sm 7877 dundee 2 24592"

|===========================================================================================================                                                                                                               | 49% ~2 s remaining     [1] "rnng bllip-sm 7877 natural-stories 2 3078"

|==============================================================================================================                                                                                                            | 51% ~2 s remaining     [1] "rnng bllip-sm 64924 bnc-brown 2 1691"
[1] "rnng bllip-sm 64924 dundee 2 24563"

|===================================================================================================================                                                                                                       | 53% ~2 s remaining     [1] "rnng bllip-sm 64924 natural-stories 2 3075"

|=====================================================================================================================                                                                                                     | 54% ~2 s remaining     [1] "rnng bllip-xs 4301 bnc-brown 2 1683"
[1] "rnng bllip-xs 4301 dundee 2 24528"

|==========================================================================================================================                                                                                                | 56% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 2 3118"

|=============================================================================================================================                                                                                             | 57% ~2 s remaining     [1] "rnng bllip-xs 28066 bnc-brown 2 1698"

|===============================================================================================================================                                                                                           | 59% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 2 24468"

|==================================================================================================================================                                                                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 2 3111"

|====================================================================================================================================                                                                                      | 61% ~2 s remaining     [1] "rnng bllip-xs 28068 bnc-brown 2 1697"

|=======================================================================================================================================                                                                                   | 62% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 2 24568"

|=========================================================================================================================================                                                                                 | 63% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 2 3080"

|============================================================================================================================================                                                                              | 64% ~2 s remaining     [1] "rnng bllip-xs 51272 bnc-brown 2 1729"

|==============================================================================================================================================                                                                            | 66% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 2 24543"

|=================================================================================================================================================                                                                         | 67% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 2 3078"

|===================================================================================================================================================                                                                       | 68% ~2 s remaining     [1] "vanilla bllip-lg 111 bnc-brown 2 1692"

|======================================================================================================================================================                                                                    | 69% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 2 24446"

|========================================================================================================================================================                                                                  | 70% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 2 3086"

|===========================================================================================================================================================                                                               | 71% ~2 s remaining     [1] "vanilla bllip-md 120 bnc-brown 2 1678"

|=============================================================================================================================================================                                                             | 72% ~1 s remaining     [1] "vanilla bllip-md 120 dundee 2 24549"

|================================================================================================================================================================                                                          | 74% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 2 3093"

|==================================================================================================================================================================                                                        | 75% ~1 s remaining     [1] "vanilla bllip-md 607 bnc-brown 2 1683"
[1] "vanilla bllip-md 922 bnc-brown 2 1679"

|=======================================================================================================================================================================                                                   | 77% ~1 s remaining     [1] "vanilla bllip-md 922 dundee 2 24590"

|==========================================================================================================================================================================                                                | 78% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 2 3098"

|============================================================================================================================================================================                                              | 79% ~1 s remaining     [1] "vanilla bllip-sm 111 bnc-brown 2 1675"
[1] "vanilla bllip-sm 111 dundee 2 24477"

|=================================================================================================================================================================================                                         | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 2 3102"

|====================================================================================================================================================================================                                      | 83% ~1 s remaining     [1] "vanilla bllip-sm 120 bnc-brown 2 1694"
[1] "vanilla bllip-sm 120 dundee 2 24621"

|=========================================================================================================================================================================================                                 | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 2 3093"

|===========================================================================================================================================================================================                               | 86% ~1 s remaining     [1] "vanilla bllip-sm 922 bnc-brown 2 1702"
[1] "vanilla bllip-sm 922 dundee 2 24517"

|================================================================================================================================================================================================                          | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 2 3057"

|===================================================================================================================================================================================================                       | 90% ~1 s remaining     [1] "vanilla bllip-xs 111 bnc-brown 2 1663"

|=====================================================================================================================================================================================================                     | 91% ~1 s remaining     [1] "vanilla bllip-xs 111 dundee 2 24552"

|========================================================================================================================================================================================================                  | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 2 3109"

|==========================================================================================================================================================================================================                | 93% ~0 s remaining     [1] "vanilla bllip-xs 120 bnc-brown 2 1711"

|=============================================================================================================================================================================================================             | 94% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 2 24599"

|===============================================================================================================================================================================================================           | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 2 3120"

|==================================================================================================================================================================================================================        | 97% ~0 s remaining     [1] "vanilla bllip-xs 922 bnc-brown 2 1689"

|====================================================================================================================================================================================================================      | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 2 24647"

|=======================================================================================================================================================================================================================   | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 2 3092"

|==========================================================================================================================================================================================================================|100% ~0 s remaining     [1] "5gram bllip-lg 1111 bnc-brown 2 1694"
[1] "5gram bllip-lg 1111 dundee 2 24547"
[1] "5gram bllip-lg 1111 natural-stories 2 3079"
[1] "5gram bllip-md 1111 bnc-brown 2 1688"
[1] "5gram bllip-md 1111 dundee 2 24574"
[1] "5gram bllip-md 1111 natural-stories 2 3121"
[1] "5gram bllip-sm 1111 bnc-brown 2 1685"
[1] "5gram bllip-sm 1111 dundee 2 24612"
[1] "5gram bllip-sm 1111 natural-stories 2 3106"
[1] "5gram bllip-xs 1111 bnc-brown 2 1675"
[1] "5gram bllip-xs 1111 dundee 2 24556"
[1] "5gram bllip-xs 1111 natural-stories 2 3108"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 2 1684"
[1] "gpt2 bllip-lg 1587139950 dundee 2 24520"
[1] "gpt2 bllip-lg 1587139950 natural-stories 2 3071"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 2 1696"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 2 24634"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 2 3096"
[1] "gpt2 bllip-md 1586986276 bnc-brown 2 1684"
[1] "gpt2 bllip-md 1586986276 dundee 2 24578"
[1] "gpt2 bllip-md 1586986276 natural-stories 2 3129"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 2 24544"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 2 3094"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 2 1720"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 2 24604"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 2 3105"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 2 1686"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 2 24507"
[1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 2 3081"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 2 1693"
[1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 2 24529"
[1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 2 3085"
[1] "rnng bllip-lg 7245 bnc-brown 2 1684"
[1] "rnng bllip-lg 7245 dundee 2 24464"
[1] "rnng bllip-lg 7245 natural-stories 2 3082"
[1] "rnng bllip-md 3602 bnc-brown 2 1700"
[1] "rnng bllip-md 3602 dundee 2 24518"

|============================================================================================                                                                                                                              | 43% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 2 3104"

|===============================================================================================                                                                                                                           | 44% ~3 s remaining     [1] "rnng bllip-md 44862 bnc-brown 2 1692"

|=================================================================================================                                                                                                                         | 45% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 2 24513"

|====================================================================================================                                                                                                                      | 46% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 2 3093"

|======================================================================================================                                                                                                                    | 47% ~3 s remaining     [1] "rnng bllip-sm 7877 bnc-brown 2 1707"

|=========================================================================================================                                                                                                                 | 48% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 2 24592"

|===========================================================================================================                                                                                                               | 49% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 2 3078"

|==============================================================================================================                                                                                                            | 51% ~2 s remaining     [1] "rnng bllip-sm 64924 bnc-brown 2 1691"

|================================================================================================================                                                                                                          | 52% ~2 s remaining     [1] "rnng bllip-sm 64924 dundee 2 24563"

|===================================================================================================================                                                                                                       | 53% ~2 s remaining     [1] "rnng bllip-sm 64924 natural-stories 2 3075"

|=====================================================================================================================                                                                                                     | 54% ~2 s remaining     [1] "rnng bllip-xs 4301 bnc-brown 2 1683"

|========================================================================================================================                                                                                                  | 55% ~2 s remaining     [1] "rnng bllip-xs 4301 dundee 2 24528"

|==========================================================================================================================                                                                                                | 56% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 2 3118"

|=============================================================================================================================                                                                                             | 57% ~2 s remaining     [1] "rnng bllip-xs 28066 bnc-brown 2 1698"

|===============================================================================================================================                                                                                           | 59% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 2 24468"

|==================================================================================================================================                                                                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 2 3111"

|====================================================================================================================================                                                                                      | 61% ~2 s remaining     [1] "rnng bllip-xs 28068 bnc-brown 2 1697"

|=======================================================================================================================================                                                                                   | 62% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 2 24568"

|=========================================================================================================================================                                                                                 | 63% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 2 3080"

|============================================================================================================================================                                                                              | 64% ~2 s remaining     [1] "rnng bllip-xs 51272 bnc-brown 2 1729"

|==============================================================================================================================================                                                                            | 66% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 2 24543"

|=================================================================================================================================================                                                                         | 67% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 2 3078"

|===================================================================================================================================================                                                                       | 68% ~2 s remaining     [1] "vanilla bllip-lg 111 bnc-brown 2 1692"

|======================================================================================================================================================                                                                    | 69% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 2 24446"

|========================================================================================================================================================                                                                  | 70% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 2 3086"

|===========================================================================================================================================================                                                               | 71% ~2 s remaining     [1] "vanilla bllip-md 120 bnc-brown 2 1678"

|=============================================================================================================================================================                                                             | 72% ~2 s remaining     [1] "vanilla bllip-md 120 dundee 2 24549"

|================================================================================================================================================================                                                          | 74% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 2 3093"

|==================================================================================================================================================================                                                        | 75% ~1 s remaining     [1] "vanilla bllip-md 607 bnc-brown 2 1683"

|=====================================================================================================================================================================                                                     | 76% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 2 1679"

|=======================================================================================================================================================================                                                   | 77% ~1 s remaining     [1] "vanilla bllip-md 922 dundee 2 24590"

|==========================================================================================================================================================================                                                | 78% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 2 3098"

|============================================================================================================================================================================                                              | 79% ~1 s remaining     [1] "vanilla bllip-sm 111 bnc-brown 2 1675"

|===============================================================================================================================================================================                                           | 80% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 2 24477"

|=================================================================================================================================================================================                                         | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 2 3102"

|====================================================================================================================================================================================                                      | 83% ~1 s remaining     [1] "vanilla bllip-sm 120 bnc-brown 2 1694"

|======================================================================================================================================================================================                                    | 84% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 2 24621"

|=========================================================================================================================================================================================                                 | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 2 3093"

|===========================================================================================================================================================================================                               | 86% ~1 s remaining     [1] "vanilla bllip-sm 922 bnc-brown 2 1702"

|==============================================================================================================================================================================================                            | 87% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 2 24517"

|================================================================================================================================================================================================                          | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 2 3057"

|===================================================================================================================================================================================================                       | 90% ~1 s remaining     [1] "vanilla bllip-xs 111 bnc-brown 2 1663"

|=====================================================================================================================================================================================================                     | 91% ~1 s remaining     [1] "vanilla bllip-xs 111 dundee 2 24552"

|========================================================================================================================================================================================================                  | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 2 3109"

|==========================================================================================================================================================================================================                | 93% ~0 s remaining     [1] "vanilla bllip-xs 120 bnc-brown 2 1711"

|=============================================================================================================================================================================================================             | 94% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 2 24599"

|===============================================================================================================================================================================================================           | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 2 3120"

|==================================================================================================================================================================================================================        | 97% ~0 s remaining     [1] "vanilla bllip-xs 922 bnc-brown 2 1689"

|====================================================================================================================================================================================================================      | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 2 24647"

|=======================================================================================================================================================================================================================   | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 2 3092"

|==========================================================================================================================================================================================================================|100% ~0 s remaining     

[1] "5gram bllip-lg 1111 bnc-brown 3 1700"
[1] "5gram bllip-lg 1111 dundee 3 24565"
[1] "5gram bllip-lg 1111 natural-stories 3 3083"
[1] "5gram bllip-md 1111 bnc-brown 3 1682"
[1] "5gram bllip-md 1111 dundee 3 24536"
[1] "5gram bllip-md 1111 natural-stories 3 3083"
[1] "5gram bllip-sm 1111 bnc-brown 3 1701"
[1] "5gram bllip-sm 1111 dundee 3 24516"
[1] "5gram bllip-sm 1111 natural-stories 3 3107"
[1] "5gram bllip-xs 1111 bnc-brown 3 1692"
[1] "5gram bllip-xs 1111 dundee 3 24576"
[1] "5gram bllip-xs 1111 natural-stories 3 3106"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 3 1669"
[1] "gpt2 bllip-lg 1587139950 dundee 3 24578"
[1] "gpt2 bllip-lg 1587139950 natural-stories 3 3143"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 3 1717"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 3 24588"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 3 3108"
[1] "gpt2 bllip-md 1586986276 bnc-brown 3 1709"
[1] "gpt2 bllip-md 1586986276 dundee 3 24460"
[1] "gpt2 bllip-md 1586986276 natural-stories 3 3103"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 3 24465"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 3 3097"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 3 1674"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 3 24515"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 3 3072"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 3 1693"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 3 24579"
[1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 3 3094"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 3 1675"
[1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 3 24516"
[1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 3 3058"
[1] "rnng bllip-lg 7245 bnc-brown 3 1671"
[1] "rnng bllip-lg 7245 dundee 3 24642"
[1] "rnng bllip-lg 7245 natural-stories 3 3085"
[1] "rnng bllip-md 3602 bnc-brown 3 1691"
[1] "rnng bllip-md 3602 dundee 3 24633"
[1] "rnng bllip-md 3602 natural-stories 3 3110"
[1] "rnng bllip-md 44862 bnc-brown 3 1691"
[1] "rnng bllip-md 44862 dundee 3 24532"

|====================================================================================================                                                                                                                      | 46% ~2 s remaining     [1] "rnng bllip-md 44862 natural-stories 3 3102"

|======================================================================================================                                                                                                                    | 47% ~2 s remaining     [1] "rnng bllip-sm 7877 bnc-brown 3 1680"

|=========================================================================================================                                                                                                                 | 48% ~2 s remaining     [1] "rnng bllip-sm 7877 dundee 3 24512"

|===========================================================================================================                                                                                                               | 49% ~2 s remaining     [1] "rnng bllip-sm 7877 natural-stories 3 3069"

|==============================================================================================================                                                                                                            | 51% ~2 s remaining     [1] "rnng bllip-sm 64924 bnc-brown 3 1665"
[1] "rnng bllip-sm 64924 dundee 3 24542"

|===================================================================================================================                                                                                                       | 53% ~2 s remaining     [1] "rnng bllip-sm 64924 natural-stories 3 3115"

|=====================================================================================================================                                                                                                     | 54% ~2 s remaining     [1] "rnng bllip-xs 4301 bnc-brown 3 1706"
[1] "rnng bllip-xs 4301 dundee 3 24485"

|==========================================================================================================================                                                                                                | 56% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 3 3121"

|=============================================================================================================================                                                                                             | 57% ~2 s remaining     [1] "rnng bllip-xs 28066 bnc-brown 3 1711"

|===============================================================================================================================                                                                                           | 59% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 3 24491"

|==================================================================================================================================                                                                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 3 3102"

|====================================================================================================================================                                                                                      | 61% ~2 s remaining     [1] "rnng bllip-xs 28068 bnc-brown 3 1697"

|=======================================================================================================================================                                                                                   | 62% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 3 24460"

|=========================================================================================================================================                                                                                 | 63% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 3 3079"

|============================================================================================================================================                                                                              | 64% ~2 s remaining     [1] "rnng bllip-xs 51272 bnc-brown 3 1699"

|==============================================================================================================================================                                                                            | 66% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 3 24585"

|=================================================================================================================================================                                                                         | 67% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 3 3077"

|===================================================================================================================================================                                                                       | 68% ~2 s remaining     [1] "vanilla bllip-lg 111 bnc-brown 3 1718"

|======================================================================================================================================================                                                                    | 69% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 3 24583"

|========================================================================================================================================================                                                                  | 70% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 3 3111"

|===========================================================================================================================================================                                                               | 71% ~2 s remaining     [1] "vanilla bllip-md 120 bnc-brown 3 1703"

|=============================================================================================================================================================                                                             | 72% ~1 s remaining     [1] "vanilla bllip-md 120 dundee 3 24546"

|================================================================================================================================================================                                                          | 74% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 3 3102"

|==================================================================================================================================================================                                                        | 75% ~1 s remaining     [1] "vanilla bllip-md 607 bnc-brown 3 1694"

|=====================================================================================================================================================================                                                     | 76% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 3 1701"

|=======================================================================================================================================================================                                                   | 77% ~1 s remaining     [1] "vanilla bllip-md 922 dundee 3 24576"

|==========================================================================================================================================================================                                                | 78% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 3 3101"

|============================================================================================================================================================================                                              | 79% ~1 s remaining     [1] "vanilla bllip-sm 111 bnc-brown 3 1695"

|===============================================================================================================================================================================                                           | 80% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 3 24577"

|=================================================================================================================================================================================                                         | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 3 3108"

|====================================================================================================================================================================================                                      | 83% ~1 s remaining     [1] "vanilla bllip-sm 120 bnc-brown 3 1695"

|======================================================================================================================================================================================                                    | 84% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 3 24485"

|=========================================================================================================================================================================================                                 | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 3 3082"

|===========================================================================================================================================================================================                               | 86% ~1 s remaining     [1] "vanilla bllip-sm 922 bnc-brown 3 1699"

|==============================================================================================================================================================================================                            | 87% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 3 24563"

|================================================================================================================================================================================================                          | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 3 3154"

|===================================================================================================================================================================================================                       | 90% ~1 s remaining     [1] "vanilla bllip-xs 111 bnc-brown 3 1718"

|=====================================================================================================================================================================================================                     | 91% ~1 s remaining     [1] "vanilla bllip-xs 111 dundee 3 24633"

|========================================================================================================================================================================================================                  | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 3 3093"

|==========================================================================================================================================================================================================                | 93% ~0 s remaining     [1] "vanilla bllip-xs 120 bnc-brown 3 1701"

|=============================================================================================================================================================================================================             | 94% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 3 24563"

|===============================================================================================================================================================================================================           | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 3 3060"

|==================================================================================================================================================================================================================        | 97% ~0 s remaining     [1] "vanilla bllip-xs 922 bnc-brown 3 1681"

|====================================================================================================================================================================================================================      | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 3 24521"

|=======================================================================================================================================================================================================================   | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 3 3080"

|==========================================================================================================================================================================================================================|100% ~0 s remaining     [1] "5gram bllip-lg 1111 bnc-brown 3 1700"
[1] "5gram bllip-lg 1111 dundee 3 24565"
[1] "5gram bllip-lg 1111 natural-stories 3 3083"
[1] "5gram bllip-md 1111 bnc-brown 3 1682"
[1] "5gram bllip-md 1111 dundee 3 24536"
[1] "5gram bllip-md 1111 natural-stories 3 3083"
[1] "5gram bllip-sm 1111 bnc-brown 3 1701"
[1] "5gram bllip-sm 1111 dundee 3 24516"
[1] "5gram bllip-sm 1111 natural-stories 3 3107"
[1] "5gram bllip-xs 1111 bnc-brown 3 1692"
[1] "5gram bllip-xs 1111 dundee 3 24576"
[1] "5gram bllip-xs 1111 natural-stories 3 3106"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 3 1669"
[1] "gpt2 bllip-lg 1587139950 dundee 3 24578"
[1] "gpt2 bllip-lg 1587139950 natural-stories 3 3143"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 3 1717"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 3 24588"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 3 3108"
[1] "gpt2 bllip-md 1586986276 bnc-brown 3 1709"
[1] "gpt2 bllip-md 1586986276 dundee 3 24460"
[1] "gpt2 bllip-md 1586986276 natural-stories 3 3103"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 3 24465"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 3 3097"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 3 1674"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 3 24515"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 3 3072"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 3 1693"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 3 24579"
[1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 3 3094"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 3 1675"
[1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 3 24516"
[1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 3 3058"
[1] "rnng bllip-lg 7245 bnc-brown 3 1671"
[1] "rnng bllip-lg 7245 dundee 3 24642"
[1] "rnng bllip-lg 7245 natural-stories 3 3085"
[1] "rnng bllip-md 3602 bnc-brown 3 1691"
[1] "rnng bllip-md 3602 dundee 3 24633"

|============================================================================================                                                                                                                              | 43% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 3 3110"

|===============================================================================================                                                                                                                           | 44% ~3 s remaining     [1] "rnng bllip-md 44862 bnc-brown 3 1691"

|=================================================================================================                                                                                                                         | 45% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 3 24532"

|====================================================================================================                                                                                                                      | 46% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 3 3102"

|======================================================================================================                                                                                                                    | 47% ~3 s remaining     [1] "rnng bllip-sm 7877 bnc-brown 3 1680"

|=========================================================================================================                                                                                                                 | 48% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 3 24512"

|===========================================================================================================                                                                                                               | 49% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 3 3069"

|==============================================================================================================                                                                                                            | 51% ~3 s remaining     [1] "rnng bllip-sm 64924 bnc-brown 3 1665"

|================================================================================================================                                                                                                          | 52% ~3 s remaining     [1] "rnng bllip-sm 64924 dundee 3 24542"

|===================================================================================================================                                                                                                       | 53% ~3 s remaining     [1] "rnng bllip-sm 64924 natural-stories 3 3115"

|=====================================================================================================================                                                                                                     | 54% ~3 s remaining     [1] "rnng bllip-xs 4301 bnc-brown 3 1706"

|========================================================================================================================                                                                                                  | 55% ~3 s remaining     [1] "rnng bllip-xs 4301 dundee 3 24485"

|==========================================================================================================================                                                                                                | 56% ~3 s remaining     [1] "rnng bllip-xs 4301 natural-stories 3 3121"

|=============================================================================================================================                                                                                             | 57% ~3 s remaining     [1] "rnng bllip-xs 28066 bnc-brown 3 1711"

|===============================================================================================================================                                                                                           | 59% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 3 24491"

|==================================================================================================================================                                                                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 3 3102"

|====================================================================================================================================                                                                                      | 61% ~2 s remaining     [1] "rnng bllip-xs 28068 bnc-brown 3 1697"

|=======================================================================================================================================                                                                                   | 62% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 3 24460"

|=========================================================================================================================================                                                                                 | 63% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 3 3079"

|============================================================================================================================================                                                                              | 64% ~2 s remaining     [1] "rnng bllip-xs 51272 bnc-brown 3 1699"

|==============================================================================================================================================                                                                            | 66% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 3 24585"

|=================================================================================================================================================                                                                         | 67% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 3 3077"

|===================================================================================================================================================                                                                       | 68% ~2 s remaining     [1] "vanilla bllip-lg 111 bnc-brown 3 1718"

|======================================================================================================================================================                                                                    | 69% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 3 24583"

|========================================================================================================================================================                                                                  | 70% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 3 3111"

|===========================================================================================================================================================                                                               | 71% ~2 s remaining     [1] "vanilla bllip-md 120 bnc-brown 3 1703"

|=============================================================================================================================================================                                                             | 72% ~2 s remaining     [1] "vanilla bllip-md 120 dundee 3 24546"

|================================================================================================================================================================                                                          | 74% ~2 s remaining     [1] "vanilla bllip-md 120 natural-stories 3 3102"

|==================================================================================================================================================================                                                        | 75% ~2 s remaining     [1] "vanilla bllip-md 607 bnc-brown 3 1694"

|=====================================================================================================================================================================                                                     | 76% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 3 1701"

|=======================================================================================================================================================================                                                   | 77% ~1 s remaining     [1] "vanilla bllip-md 922 dundee 3 24576"

|==========================================================================================================================================================================                                                | 78% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 3 3101"

|============================================================================================================================================================================                                              | 79% ~1 s remaining     [1] "vanilla bllip-sm 111 bnc-brown 3 1695"

|===============================================================================================================================================================================                                           | 80% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 3 24577"

|=================================================================================================================================================================================                                         | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 3 3108"

|====================================================================================================================================================================================                                      | 83% ~1 s remaining     [1] "vanilla bllip-sm 120 bnc-brown 3 1695"

|======================================================================================================================================================================================                                    | 84% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 3 24485"

|=========================================================================================================================================================================================                                 | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 3 3082"

|===========================================================================================================================================================================================                               | 86% ~1 s remaining     [1] "vanilla bllip-sm 922 bnc-brown 3 1699"

|==============================================================================================================================================================================================                            | 87% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 3 24563"

|================================================================================================================================================================================================                          | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 3 3154"

|===================================================================================================================================================================================================                       | 90% ~1 s remaining     [1] "vanilla bllip-xs 111 bnc-brown 3 1718"

|=====================================================================================================================================================================================================                     | 91% ~1 s remaining     [1] "vanilla bllip-xs 111 dundee 3 24633"

|========================================================================================================================================================================================================                  | 92% ~1 s remaining     [1] "vanilla bllip-xs 111 natural-stories 3 3093"

|==========================================================================================================================================================================================================                | 93% ~0 s remaining     [1] "vanilla bllip-xs 120 bnc-brown 3 1701"

|=============================================================================================================================================================================================================             | 94% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 3 24563"

|===============================================================================================================================================================================================================           | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 3 3060"

|==================================================================================================================================================================================================================        | 97% ~0 s remaining     [1] "vanilla bllip-xs 922 bnc-brown 3 1681"

|====================================================================================================================================================================================================================      | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 3 24521"

|=======================================================================================================================================================================================================================   | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 3 3080"

|==========================================================================================================================================================================================================================|100% ~0 s remaining     

[1] "5gram bllip-lg 1111 bnc-brown 4 1711"
[1] "5gram bllip-lg 1111 dundee 4 24542"
[1] "5gram bllip-lg 1111 natural-stories 4 3091"
[1] "5gram bllip-md 1111 bnc-brown 4 1694"
[1] "5gram bllip-md 1111 dundee 4 24543"
[1] "5gram bllip-md 1111 natural-stories 4 3083"
[1] "5gram bllip-sm 1111 bnc-brown 4 1723"
[1] "5gram bllip-sm 1111 dundee 4 24578"
[1] "5gram bllip-sm 1111 natural-stories 4 3102"
[1] "5gram bllip-xs 1111 bnc-brown 4 1688"
[1] "5gram bllip-xs 1111 dundee 4 24619"
[1] "5gram bllip-xs 1111 natural-stories 4 3112"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 4 1690"
[1] "gpt2 bllip-lg 1587139950 dundee 4 24554"
[1] "gpt2 bllip-lg 1587139950 natural-stories 4 3082"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 4 1657"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 4 24590"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 4 3067"
[1] "gpt2 bllip-md 1586986276 bnc-brown 4 1676"
[1] "gpt2 bllip-md 1586986276 dundee 4 24567"
[1] "gpt2 bllip-md 1586986276 natural-stories 4 3117"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 4 24552"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 4 3092"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 4 1670"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 4 24513"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 4 3112"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 4 1687"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 4 24410"
[1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 4 3103"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 4 1685"
[1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 4 24576"
[1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 4 3097"
[1] "rnng bllip-lg 7245 bnc-brown 4 1698"
[1] "rnng bllip-lg 7245 dundee 4 24565"
[1] "rnng bllip-lg 7245 natural-stories 4 3102"
[1] "rnng bllip-md 3602 bnc-brown 4 1702"
[1] "rnng bllip-md 3602 dundee 4 24595"

|============================================================================================                                                                                                                              | 43% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 4 3090"

|===============================================================================================                                                                                                                           | 44% ~3 s remaining     [1] "rnng bllip-md 44862 bnc-brown 4 1683"

|=================================================================================================                                                                                                                         | 45% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 4 24598"

|====================================================================================================                                                                                                                      | 46% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 4 3065"

|======================================================================================================                                                                                                                    | 47% ~3 s remaining     [1] "rnng bllip-sm 7877 bnc-brown 4 1672"

|=========================================================================================================                                                                                                                 | 48% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 4 24600"

|===========================================================================================================                                                                                                               | 49% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 4 3104"

|==============================================================================================================                                                                                                            | 51% ~3 s remaining     [1] "rnng bllip-sm 64924 bnc-brown 4 1695"

|================================================================================================================                                                                                                          | 52% ~3 s remaining     [1] "rnng bllip-sm 64924 dundee 4 24481"

|===================================================================================================================                                                                                                       | 53% ~3 s remaining     [1] "rnng bllip-sm 64924 natural-stories 4 3118"

|=====================================================================================================================                                                                                                     | 54% ~3 s remaining     [1] "rnng bllip-xs 4301 bnc-brown 4 1692"

|========================================================================================================================                                                                                                  | 55% ~3 s remaining     [1] "rnng bllip-xs 4301 dundee 4 24530"

|==========================================================================================================================                                                                                                | 56% ~3 s remaining     [1] "rnng bllip-xs 4301 natural-stories 4 3128"

|=============================================================================================================================                                                                                             | 57% ~3 s remaining     [1] "rnng bllip-xs 28066 bnc-brown 4 1695"

|===============================================================================================================================                                                                                           | 59% ~3 s remaining     [1] "rnng bllip-xs 28066 dundee 4 24585"

|==================================================================================================================================                                                                                        | 60% ~3 s remaining     [1] "rnng bllip-xs 28066 natural-stories 4 3074"

|====================================================================================================================================                                                                                      | 61% ~2 s remaining     [1] "rnng bllip-xs 28068 bnc-brown 4 1684"

|=======================================================================================================================================                                                                                   | 62% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 4 24692"

|=========================================================================================================================================                                                                                 | 63% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 4 3115"

|============================================================================================================================================                                                                              | 64% ~2 s remaining     [1] "rnng bllip-xs 51272 bnc-brown 4 1686"
[1] "rnng bllip-xs 51272 dundee 4 24598"

|=================================================================================================================================================                                                                         | 67% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 4 3124"

|===================================================================================================================================================                                                                       | 68% ~2 s remaining     [1] "vanilla bllip-lg 111 bnc-brown 4 1690"
[1] "vanilla bllip-lg 111 dundee 4 24451"

|========================================================================================================================================================                                                                  | 70% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 4 3123"

|===========================================================================================================================================================                                                               | 71% ~2 s remaining     [1] "vanilla bllip-md 120 bnc-brown 4 1700"
[1] "vanilla bllip-md 120 dundee 4 24519"

|================================================================================================================================================================                                                          | 74% ~2 s remaining     [1] "vanilla bllip-md 120 natural-stories 4 3092"

|==================================================================================================================================================================                                                        | 75% ~2 s remaining     [1] "vanilla bllip-md 607 bnc-brown 4 1721"

|=====================================================================================================================================================================                                                     | 76% ~2 s remaining     [1] "vanilla bllip-md 922 bnc-brown 4 1682"

|=======================================================================================================================================================================                                                   | 77% ~2 s remaining     [1] "vanilla bllip-md 922 dundee 4 24529"

|==========================================================================================================================================================================                                                | 78% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 4 3102"

|============================================================================================================================================================================                                              | 79% ~1 s remaining     [1] "vanilla bllip-sm 111 bnc-brown 4 1699"

|===============================================================================================================================================================================                                           | 80% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 4 24527"

|=================================================================================================================================================================================                                         | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 4 3071"

|====================================================================================================================================================================================                                      | 83% ~1 s remaining     [1] "vanilla bllip-sm 120 bnc-brown 4 1690"

|======================================================================================================================================================================================                                    | 84% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 4 24584"

|=========================================================================================================================================================================================                                 | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 4 3089"

|===========================================================================================================================================================================================                               | 86% ~1 s remaining     [1] "vanilla bllip-sm 922 bnc-brown 4 1685"

|==============================================================================================================================================================================================                            | 87% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 4 24568"

|================================================================================================================================================================================================                          | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 4 3072"

|===================================================================================================================================================================================================                       | 90% ~1 s remaining     [1] "vanilla bllip-xs 111 bnc-brown 4 1708"

|=====================================================================================================================================================================================================                     | 91% ~1 s remaining     [1] "vanilla bllip-xs 111 dundee 4 24442"

|========================================================================================================================================================================================================                  | 92% ~1 s remaining     [1] "vanilla bllip-xs 111 natural-stories 4 3110"

|==========================================================================================================================================================================================================                | 93% ~0 s remaining     [1] "vanilla bllip-xs 120 bnc-brown 4 1708"

|=============================================================================================================================================================================================================             | 94% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 4 24506"

|===============================================================================================================================================================================================================           | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 4 3054"

|==================================================================================================================================================================================================================        | 97% ~0 s remaining     [1] "vanilla bllip-xs 922 bnc-brown 4 1710"

|====================================================================================================================================================================================================================      | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 4 24476"

|=======================================================================================================================================================================================================================   | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 4 3093"

|==========================================================================================================================================================================================================================|100% ~0 s remaining     [1] "5gram bllip-lg 1111 bnc-brown 4 1711"
[1] "5gram bllip-lg 1111 dundee 4 24542"
[1] "5gram bllip-lg 1111 natural-stories 4 3091"
[1] "5gram bllip-md 1111 bnc-brown 4 1694"
[1] "5gram bllip-md 1111 dundee 4 24543"
[1] "5gram bllip-md 1111 natural-stories 4 3083"
[1] "5gram bllip-sm 1111 bnc-brown 4 1723"
[1] "5gram bllip-sm 1111 dundee 4 24578"
[1] "5gram bllip-sm 1111 natural-stories 4 3102"
[1] "5gram bllip-xs 1111 bnc-brown 4 1688"
[1] "5gram bllip-xs 1111 dundee 4 24619"
[1] "5gram bllip-xs 1111 natural-stories 4 3112"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 4 1690"
[1] "gpt2 bllip-lg 1587139950 dundee 4 24554"
[1] "gpt2 bllip-lg 1587139950 natural-stories 4 3082"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 4 1657"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 4 24590"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 4 3067"
[1] "gpt2 bllip-md 1586986276 bnc-brown 4 1676"
[1] "gpt2 bllip-md 1586986276 dundee 4 24567"
[1] "gpt2 bllip-md 1586986276 natural-stories 4 3117"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 4 24552"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 4 3092"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 4 1670"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 4 24513"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 4 3112"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 4 1687"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 4 24410"
[1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 4 3103"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 4 1685"
[1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 4 24576"
[1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 4 3097"
[1] "rnng bllip-lg 7245 bnc-brown 4 1698"
[1] "rnng bllip-lg 7245 dundee 4 24565"
[1] "rnng bllip-lg 7245 natural-stories 4 3102"
[1] "rnng bllip-md 3602 bnc-brown 4 1702"
[1] "rnng bllip-md 3602 dundee 4 24595"
[1] "rnng bllip-md 3602 natural-stories 4 3090"
[1] "rnng bllip-md 44862 bnc-brown 4 1683"

|=================================================================================================                                                                                                                         | 45% ~2 s remaining     [1] "rnng bllip-md 44862 dundee 4 24598"

|====================================================================================================                                                                                                                      | 46% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 4 3065"

|======================================================================================================                                                                                                                    | 47% ~3 s remaining     [1] "rnng bllip-sm 7877 bnc-brown 4 1672"

|=========================================================================================================                                                                                                                 | 48% ~2 s remaining     [1] "rnng bllip-sm 7877 dundee 4 24600"

|===========================================================================================================                                                                                                               | 49% ~2 s remaining     [1] "rnng bllip-sm 7877 natural-stories 4 3104"

|==============================================================================================================                                                                                                            | 51% ~2 s remaining     [1] "rnng bllip-sm 64924 bnc-brown 4 1695"

|================================================================================================================                                                                                                          | 52% ~2 s remaining     [1] "rnng bllip-sm 64924 dundee 4 24481"

|===================================================================================================================                                                                                                       | 53% ~2 s remaining     [1] "rnng bllip-sm 64924 natural-stories 4 3118"

|=====================================================================================================================                                                                                                     | 54% ~2 s remaining     [1] "rnng bllip-xs 4301 bnc-brown 4 1692"

|========================================================================================================================                                                                                                  | 55% ~2 s remaining     [1] "rnng bllip-xs 4301 dundee 4 24530"

|==========================================================================================================================                                                                                                | 56% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 4 3128"

|=============================================================================================================================                                                                                             | 57% ~2 s remaining     [1] "rnng bllip-xs 28066 bnc-brown 4 1695"

|===============================================================================================================================                                                                                           | 59% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 4 24585"

|==================================================================================================================================                                                                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 4 3074"

|====================================================================================================================================                                                                                      | 61% ~2 s remaining     [1] "rnng bllip-xs 28068 bnc-brown 4 1684"

|=======================================================================================================================================                                                                                   | 62% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 4 24692"

|=========================================================================================================================================                                                                                 | 63% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 4 3115"

|============================================================================================================================================                                                                              | 64% ~2 s remaining     [1] "rnng bllip-xs 51272 bnc-brown 4 1686"

|==============================================================================================================================================                                                                            | 66% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 4 24598"

|=================================================================================================================================================                                                                         | 67% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 4 3124"

|===================================================================================================================================================                                                                       | 68% ~2 s remaining     [1] "vanilla bllip-lg 111 bnc-brown 4 1690"

|======================================================================================================================================================                                                                    | 69% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 4 24451"

|========================================================================================================================================================                                                                  | 70% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 4 3123"

|===========================================================================================================================================================                                                               | 71% ~2 s remaining     [1] "vanilla bllip-md 120 bnc-brown 4 1700"
[1] "vanilla bllip-md 120 dundee 4 24519"

|================================================================================================================================================================                                                          | 74% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 4 3092"

|==================================================================================================================================================================                                                        | 75% ~1 s remaining     [1] "vanilla bllip-md 607 bnc-brown 4 1721"

|=====================================================================================================================================================================                                                     | 76% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 4 1682"

|=======================================================================================================================================================================                                                   | 77% ~1 s remaining     [1] "vanilla bllip-md 922 dundee 4 24529"

|==========================================================================================================================================================================                                                | 78% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 4 3102"

|============================================================================================================================================================================                                              | 79% ~1 s remaining     [1] "vanilla bllip-sm 111 bnc-brown 4 1699"

|===============================================================================================================================================================================                                           | 80% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 4 24527"

|=================================================================================================================================================================================                                         | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 4 3071"

|====================================================================================================================================================================================                                      | 83% ~1 s remaining     [1] "vanilla bllip-sm 120 bnc-brown 4 1690"

|======================================================================================================================================================================================                                    | 84% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 4 24584"

|=========================================================================================================================================================================================                                 | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 4 3089"

|===========================================================================================================================================================================================                               | 86% ~1 s remaining     [1] "vanilla bllip-sm 922 bnc-brown 4 1685"

|==============================================================================================================================================================================================                            | 87% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 4 24568"

|================================================================================================================================================================================================                          | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 4 3072"

|===================================================================================================================================================================================================                       | 90% ~1 s remaining     [1] "vanilla bllip-xs 111 bnc-brown 4 1708"

|=====================================================================================================================================================================================================                     | 91% ~1 s remaining     [1] "vanilla bllip-xs 111 dundee 4 24442"

|========================================================================================================================================================================================================                  | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 4 3110"

|==========================================================================================================================================================================================================                | 93% ~0 s remaining     [1] "vanilla bllip-xs 120 bnc-brown 4 1708"

|=============================================================================================================================================================================================================             | 94% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 4 24506"

|===============================================================================================================================================================================================================           | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 4 3054"

|==================================================================================================================================================================================================================        | 97% ~0 s remaining     [1] "vanilla bllip-xs 922 bnc-brown 4 1710"

|====================================================================================================================================================================================================================      | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 4 24476"

|=======================================================================================================================================================================================================================   | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 4 3093"

|==========================================================================================================================================================================================================================|100% ~0 s remaining     

[1] "5gram bllip-lg 1111 bnc-brown 5 1693"
[1] "5gram bllip-lg 1111 dundee 5 24514"
[1] "5gram bllip-lg 1111 natural-stories 5 3118"
[1] "5gram bllip-md 1111 bnc-brown 5 1679"
[1] "5gram bllip-md 1111 dundee 5 24535"
[1] "5gram bllip-md 1111 natural-stories 5 3103"
[1] "5gram bllip-sm 1111 bnc-brown 5 1686"
[1] "5gram bllip-sm 1111 dundee 5 24453"
[1] "5gram bllip-sm 1111 natural-stories 5 3105"
[1] "5gram bllip-xs 1111 bnc-brown 5 1684"
[1] "5gram bllip-xs 1111 dundee 5 24603"
[1] "5gram bllip-xs 1111 natural-stories 5 3103"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 5 1683"
[1] "gpt2 bllip-lg 1587139950 dundee 5 24499"
[1] "gpt2 bllip-lg 1587139950 natural-stories 5 3112"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 5 1691"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 5 24514"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 5 3126"
[1] "gpt2 bllip-md 1586986276 bnc-brown 5 1673"
[1] "gpt2 bllip-md 1586986276 dundee 5 24572"
[1] "gpt2 bllip-md 1586986276 natural-stories 5 3077"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 5 24484"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 5 3098"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 5 1700"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 5 24456"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 5 3106"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 5 1703"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 5 24613"
[1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 5 3109"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 5 1709"
[1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 5 24532"
[1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 5 3116"
[1] "rnng bllip-lg 7245 bnc-brown 5 1675"
[1] "rnng bllip-lg 7245 dundee 5 24538"
[1] "rnng bllip-lg 7245 natural-stories 5 3101"
[1] "rnng bllip-md 3602 bnc-brown 5 1686"
[1] "rnng bllip-md 3602 dundee 5 24466"
[1] "rnng bllip-md 3602 natural-stories 5 3108"
[1] "rnng bllip-md 44862 bnc-brown 5 1715"
[1] "rnng bllip-md 44862 dundee 5 24574"

|====================================================================================================                                                                                                                      | 46% ~2 s remaining     [1] "rnng bllip-md 44862 natural-stories 5 3114"

|======================================================================================================                                                                                                                    | 47% ~2 s remaining     [1] "rnng bllip-sm 7877 bnc-brown 5 1694"
[1] "rnng bllip-sm 7877 dundee 5 24538"

|===========================================================================================================                                                                                                               | 49% ~2 s remaining     [1] "rnng bllip-sm 7877 natural-stories 5 3087"

|==============================================================================================================                                                                                                            | 51% ~2 s remaining     [1] "rnng bllip-sm 64924 bnc-brown 5 1701"
[1] "rnng bllip-sm 64924 dundee 5 24622"

|===================================================================================================================                                                                                                       | 53% ~2 s remaining     [1] "rnng bllip-sm 64924 natural-stories 5 3120"

|=====================================================================================================================                                                                                                     | 54% ~2 s remaining     [1] "rnng bllip-xs 4301 bnc-brown 5 1713"
[1] "rnng bllip-xs 4301 dundee 5 24527"

|==========================================================================================================================                                                                                                | 56% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 5 3113"

|=============================================================================================================================                                                                                             | 57% ~2 s remaining     [1] "rnng bllip-xs 28066 bnc-brown 5 1694"
[1] "rnng bllip-xs 28066 dundee 5 24483"

|==================================================================================================================================                                                                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 5 3094"

|====================================================================================================================================                                                                                      | 61% ~2 s remaining     [1] "rnng bllip-xs 28068 bnc-brown 5 1666"
[1] "rnng bllip-xs 28068 dundee 5 24479"

|=========================================================================================================================================                                                                                 | 63% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 5 3129"

|============================================================================================================================================                                                                              | 64% ~2 s remaining     [1] "rnng bllip-xs 51272 bnc-brown 5 1676"

|==============================================================================================================================================                                                                            | 66% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 5 24550"

|=================================================================================================================================================                                                                         | 67% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 5 3118"

|===================================================================================================================================================                                                                       | 68% ~2 s remaining     [1] "vanilla bllip-lg 111 bnc-brown 5 1705"

|======================================================================================================================================================                                                                    | 69% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 5 24574"

|========================================================================================================================================================                                                                  | 70% ~1 s remaining     [1] "vanilla bllip-lg 111 natural-stories 5 3086"

|===========================================================================================================================================================                                                               | 71% ~1 s remaining     [1] "vanilla bllip-md 120 bnc-brown 5 1706"
[1] "vanilla bllip-md 120 dundee 5 24536"

|================================================================================================================================================================                                                          | 74% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 5 3077"

|==================================================================================================================================================================                                                        | 75% ~1 s remaining     [1] "vanilla bllip-md 607 bnc-brown 5 1669"

|=====================================================================================================================================================================                                                     | 76% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 5 1681"

|=======================================================================================================================================================================                                                   | 77% ~1 s remaining     [1] "vanilla bllip-md 922 dundee 5 24516"

|==========================================================================================================================================================================                                                | 78% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 5 3109"

|============================================================================================================================================================================                                              | 79% ~1 s remaining     [1] "vanilla bllip-sm 111 bnc-brown 5 1712"
[1] "vanilla bllip-sm 111 dundee 5 24582"

|=================================================================================================================================================================================                                         | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 5 3117"

|====================================================================================================================================================================================                                      | 83% ~1 s remaining     [1] "vanilla bllip-sm 120 bnc-brown 5 1698"
[1] "vanilla bllip-sm 120 dundee 5 24623"

|=========================================================================================================================================================================================                                 | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 5 3106"

|===========================================================================================================================================================================================                               | 86% ~1 s remaining     [1] "vanilla bllip-sm 922 bnc-brown 5 1677"

|==============================================================================================================================================================================================                            | 87% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 5 24451"

|================================================================================================================================================================================================                          | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 5 3112"

|===================================================================================================================================================================================================                       | 90% ~1 s remaining     [1] "vanilla bllip-xs 111 bnc-brown 5 1688"
[1] "vanilla bllip-xs 111 dundee 5 24644"

|========================================================================================================================================================================================================                  | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 5 3088"

|==========================================================================================================================================================================================================                | 93% ~0 s remaining     [1] "vanilla bllip-xs 120 bnc-brown 5 1696"
[1] "vanilla bllip-xs 120 dundee 5 24646"

|===============================================================================================================================================================================================================           | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 5 3117"

|==================================================================================================================================================================================================================        | 97% ~0 s remaining     [1] "vanilla bllip-xs 922 bnc-brown 5 1676"
[1] "vanilla bllip-xs 922 dundee 5 24517"

|=======================================================================================================================================================================================================================   | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 5 3117"

|==========================================================================================================================================================================================================================|100% ~0 s remaining     [1] "5gram bllip-lg 1111 bnc-brown 5 1693"
[1] "5gram bllip-lg 1111 dundee 5 24514"
[1] "5gram bllip-lg 1111 natural-stories 5 3118"
[1] "5gram bllip-md 1111 bnc-brown 5 1679"
[1] "5gram bllip-md 1111 dundee 5 24535"
[1] "5gram bllip-md 1111 natural-stories 5 3103"
[1] "5gram bllip-sm 1111 bnc-brown 5 1686"
[1] "5gram bllip-sm 1111 dundee 5 24453"
[1] "5gram bllip-sm 1111 natural-stories 5 3105"
[1] "5gram bllip-xs 1111 bnc-brown 5 1684"
[1] "5gram bllip-xs 1111 dundee 5 24603"
[1] "5gram bllip-xs 1111 natural-stories 5 3103"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 5 1683"
[1] "gpt2 bllip-lg 1587139950 dundee 5 24499"
[1] "gpt2 bllip-lg 1587139950 natural-stories 5 3112"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 5 1691"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 5 24514"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 5 3126"
[1] "gpt2 bllip-md 1586986276 bnc-brown 5 1673"
[1] "gpt2 bllip-md 1586986276 dundee 5 24572"
[1] "gpt2 bllip-md 1586986276 natural-stories 5 3077"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 5 24484"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 5 3098"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 5 1700"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 5 24456"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 5 3106"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 5 1703"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 5 24613"
[1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 5 3109"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 5 1709"
[1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 5 24532"
[1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 5 3116"
[1] "rnng bllip-lg 7245 bnc-brown 5 1675"
[1] "rnng bllip-lg 7245 dundee 5 24538"
[1] "rnng bllip-lg 7245 natural-stories 5 3101"
[1] "rnng bllip-md 3602 bnc-brown 5 1686"
[1] "rnng bllip-md 3602 dundee 5 24466"

|============================================================================================                                                                                                                              | 43% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 5 3108"

|===============================================================================================                                                                                                                           | 44% ~3 s remaining     [1] "rnng bllip-md 44862 bnc-brown 5 1715"

|=================================================================================================                                                                                                                         | 45% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 5 24574"

|====================================================================================================                                                                                                                      | 46% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 5 3114"

|======================================================================================================                                                                                                                    | 47% ~3 s remaining     [1] "rnng bllip-sm 7877 bnc-brown 5 1694"

|=========================================================================================================                                                                                                                 | 48% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 5 24538"

|===========================================================================================================                                                                                                               | 49% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 5 3087"

|==============================================================================================================                                                                                                            | 51% ~3 s remaining     [1] "rnng bllip-sm 64924 bnc-brown 5 1701"

|================================================================================================================                                                                                                          | 52% ~2 s remaining     [1] "rnng bllip-sm 64924 dundee 5 24622"

|===================================================================================================================                                                                                                       | 53% ~2 s remaining     [1] "rnng bllip-sm 64924 natural-stories 5 3120"

|=====================================================================================================================                                                                                                     | 54% ~2 s remaining     [1] "rnng bllip-xs 4301 bnc-brown 5 1713"

|========================================================================================================================                                                                                                  | 55% ~2 s remaining     [1] "rnng bllip-xs 4301 dundee 5 24527"

|==========================================================================================================================                                                                                                | 56% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 5 3113"

|=============================================================================================================================                                                                                             | 57% ~2 s remaining     [1] "rnng bllip-xs 28066 bnc-brown 5 1694"

|===============================================================================================================================                                                                                           | 59% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 5 24483"

|==================================================================================================================================                                                                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 5 3094"

|====================================================================================================================================                                                                                      | 61% ~2 s remaining     [1] "rnng bllip-xs 28068 bnc-brown 5 1666"

|=======================================================================================================================================                                                                                   | 62% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 5 24479"

|=========================================================================================================================================                                                                                 | 63% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 5 3129"

|============================================================================================================================================                                                                              | 64% ~2 s remaining     [1] "rnng bllip-xs 51272 bnc-brown 5 1676"

|==============================================================================================================================================                                                                            | 66% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 5 24550"

|=================================================================================================================================================                                                                         | 67% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 5 3118"

|===================================================================================================================================================                                                                       | 68% ~2 s remaining     [1] "vanilla bllip-lg 111 bnc-brown 5 1705"

|======================================================================================================================================================                                                                    | 69% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 5 24574"

|========================================================================================================================================================                                                                  | 70% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 5 3086"

|===========================================================================================================================================================                                                               | 71% ~2 s remaining     [1] "vanilla bllip-md 120 bnc-brown 5 1706"

|=============================================================================================================================================================                                                             | 72% ~2 s remaining     [1] "vanilla bllip-md 120 dundee 5 24536"

|================================================================================================================================================================                                                          | 74% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 5 3077"

|==================================================================================================================================================================                                                        | 75% ~1 s remaining     [1] "vanilla bllip-md 607 bnc-brown 5 1669"

|=====================================================================================================================================================================                                                     | 76% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 5 1681"

|=======================================================================================================================================================================                                                   | 77% ~1 s remaining     [1] "vanilla bllip-md 922 dundee 5 24516"

|==========================================================================================================================================================================                                                | 78% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 5 3109"

|============================================================================================================================================================================                                              | 79% ~1 s remaining     [1] "vanilla bllip-sm 111 bnc-brown 5 1712"

|===============================================================================================================================================================================                                           | 80% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 5 24582"

|=================================================================================================================================================================================                                         | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 5 3117"

|====================================================================================================================================================================================                                      | 83% ~1 s remaining     [1] "vanilla bllip-sm 120 bnc-brown 5 1698"

|======================================================================================================================================================================================                                    | 84% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 5 24623"

|=========================================================================================================================================================================================                                 | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 5 3106"

|===========================================================================================================================================================================================                               | 86% ~1 s remaining     [1] "vanilla bllip-sm 922 bnc-brown 5 1677"

|==============================================================================================================================================================================================                            | 87% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 5 24451"

|================================================================================================================================================================================================                          | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 5 3112"

|===================================================================================================================================================================================================                       | 90% ~1 s remaining     [1] "vanilla bllip-xs 111 bnc-brown 5 1688"
[1] "vanilla bllip-xs 111 dundee 5 24644"

|========================================================================================================================================================================================================                  | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 5 3088"

|==========================================================================================================================================================================================================                | 93% ~0 s remaining     [1] "vanilla bllip-xs 120 bnc-brown 5 1696"

|=============================================================================================================================================================================================================             | 94% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 5 24646"

|===============================================================================================================================================================================================================           | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 5 3117"

|==================================================================================================================================================================================================================        | 97% ~0 s remaining     [1] "vanilla bllip-xs 922 bnc-brown 5 1676"

|====================================================================================================================================================================================================================      | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 5 24517"

|=======================================================================================================================================================================================================================   | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 5 3117"

|==========================================================================================================================================================================================================================|100% ~0 s remaining     

[1] "5gram bllip-lg 1111 bnc-brown 6 1702"
[1] "5gram bllip-lg 1111 dundee 6 24564"
[1] "5gram bllip-lg 1111 natural-stories 6 3115"
[1] "5gram bllip-md 1111 bnc-brown 6 1700"
[1] "5gram bllip-md 1111 dundee 6 24520"
[1] "5gram bllip-md 1111 natural-stories 6 3120"
[1] "5gram bllip-sm 1111 bnc-brown 6 1660"
[1] "5gram bllip-sm 1111 dundee 6 24623"
[1] "5gram bllip-sm 1111 natural-stories 6 3113"
[1] "5gram bllip-xs 1111 bnc-brown 6 1693"
[1] "5gram bllip-xs 1111 dundee 6 24566"
[1] "5gram bllip-xs 1111 natural-stories 6 3099"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 6 1680"
[1] "gpt2 bllip-lg 1587139950 dundee 6 24564"
[1] "gpt2 bllip-lg 1587139950 natural-stories 6 3083"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 6 1702"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 6 24467"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 6 3082"
[1] "gpt2 bllip-md 1586986276 bnc-brown 6 1710"
[1] "gpt2 bllip-md 1586986276 dundee 6 24555"
[1] "gpt2 bllip-md 1586986276 natural-stories 6 3111"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 6 24579"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 6 3109"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 6 1695"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 6 24582"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 6 3083"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 6 1685"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 6 24606"
[1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 6 3085"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 6 1678"
[1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 6 24548"
[1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 6 3119"
[1] "rnng bllip-lg 7245 bnc-brown 6 1695"
[1] "rnng bllip-lg 7245 dundee 6 24593"
[1] "rnng bllip-lg 7245 natural-stories 6 3086"
[1] "rnng bllip-md 3602 bnc-brown 6 1683"
[1] "rnng bllip-md 3602 dundee 6 24526"
[1] "rnng bllip-md 3602 natural-stories 6 3062"
[1] "rnng bllip-md 44862 bnc-brown 6 1696"
[1] "rnng bllip-md 44862 dundee 6 24565"

|====================================================================================================                                                                                                                      | 46% ~2 s remaining     [1] "rnng bllip-md 44862 natural-stories 6 3098"

|======================================================================================================                                                                                                                    | 47% ~2 s remaining     [1] "rnng bllip-sm 7877 bnc-brown 6 1684"
[1] "rnng bllip-sm 7877 dundee 6 24499"

|===========================================================================================================                                                                                                               | 49% ~2 s remaining     [1] "rnng bllip-sm 7877 natural-stories 6 3115"

|==============================================================================================================                                                                                                            | 51% ~2 s remaining     [1] "rnng bllip-sm 64924 bnc-brown 6 1710"
[1] "rnng bllip-sm 64924 dundee 6 24602"

|===================================================================================================================                                                                                                       | 53% ~2 s remaining     [1] "rnng bllip-sm 64924 natural-stories 6 3093"

|=====================================================================================================================                                                                                                     | 54% ~2 s remaining     [1] "rnng bllip-xs 4301 bnc-brown 6 1675"
[1] "rnng bllip-xs 4301 dundee 6 24533"

|==========================================================================================================================                                                                                                | 56% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 6 3051"

|=============================================================================================================================                                                                                             | 57% ~2 s remaining     [1] "rnng bllip-xs 28066 bnc-brown 6 1707"
[1] "rnng bllip-xs 28066 dundee 6 24583"

|==================================================================================================================================                                                                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 6 3110"

|====================================================================================================================================                                                                                      | 61% ~2 s remaining     [1] "rnng bllip-xs 28068 bnc-brown 6 1678"

|=======================================================================================================================================                                                                                   | 62% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 6 24490"

|=========================================================================================================================================                                                                                 | 63% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 6 3110"

|============================================================================================================================================                                                                              | 64% ~2 s remaining     [1] "rnng bllip-xs 51272 bnc-brown 6 1676"

|==============================================================================================================================================                                                                            | 66% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 6 24501"

|=================================================================================================================================================                                                                         | 67% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 6 3080"

|===================================================================================================================================================                                                                       | 68% ~2 s remaining     [1] "vanilla bllip-lg 111 bnc-brown 6 1678"
[1] "vanilla bllip-lg 111 dundee 6 24526"

|========================================================================================================================================================                                                                  | 70% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 6 3096"

|===========================================================================================================================================================                                                               | 71% ~2 s remaining     [1] "vanilla bllip-md 120 bnc-brown 6 1691"

|=============================================================================================================================================================                                                             | 72% ~2 s remaining     [1] "vanilla bllip-md 120 dundee 6 24583"

|================================================================================================================================================================                                                          | 74% ~2 s remaining     [1] "vanilla bllip-md 120 natural-stories 6 3079"

|==================================================================================================================================================================                                                        | 75% ~2 s remaining     [1] "vanilla bllip-md 607 bnc-brown 6 1699"

|=====================================================================================================================================================================                                                     | 76% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 6 1696"

|=======================================================================================================================================================================                                                   | 77% ~1 s remaining     [1] "vanilla bllip-md 922 dundee 6 24614"

|==========================================================================================================================================================================                                                | 78% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 6 3092"

|============================================================================================================================================================================                                              | 79% ~1 s remaining     [1] "vanilla bllip-sm 111 bnc-brown 6 1695"

|===============================================================================================================================================================================                                           | 80% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 6 24613"

|=================================================================================================================================================================================                                         | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 6 3100"

|====================================================================================================================================================================================                                      | 83% ~1 s remaining     [1] "vanilla bllip-sm 120 bnc-brown 6 1670"
[1] "vanilla bllip-sm 120 dundee 6 24499"

|=========================================================================================================================================================================================                                 | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 6 3098"

|===========================================================================================================================================================================================                               | 86% ~1 s remaining     [1] "vanilla bllip-sm 922 bnc-brown 6 1667"

|==============================================================================================================================================================================================                            | 87% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 6 24550"

|================================================================================================================================================================================================                          | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 6 3111"

|===================================================================================================================================================================================================                       | 90% ~1 s remaining     [1] "vanilla bllip-xs 111 bnc-brown 6 1700"
[1] "vanilla bllip-xs 111 dundee 6 24474"

|========================================================================================================================================================================================================                  | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 6 3132"

|==========================================================================================================================================================================================================                | 93% ~0 s remaining     [1] "vanilla bllip-xs 120 bnc-brown 6 1667"

|=============================================================================================================================================================================================================             | 94% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 6 24510"

|===============================================================================================================================================================================================================           | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 6 3135"

|==================================================================================================================================================================================================================        | 97% ~0 s remaining     [1] "vanilla bllip-xs 922 bnc-brown 6 1700"

|====================================================================================================================================================================================================================      | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 6 24500"

|=======================================================================================================================================================================================================================   | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 6 3082"

|==========================================================================================================================================================================================================================|100% ~0 s remaining     [1] "5gram bllip-lg 1111 bnc-brown 6 1702"
[1] "5gram bllip-lg 1111 dundee 6 24564"
[1] "5gram bllip-lg 1111 natural-stories 6 3115"
[1] "5gram bllip-md 1111 bnc-brown 6 1700"
[1] "5gram bllip-md 1111 dundee 6 24520"
[1] "5gram bllip-md 1111 natural-stories 6 3120"
[1] "5gram bllip-sm 1111 bnc-brown 6 1660"
[1] "5gram bllip-sm 1111 dundee 6 24623"
[1] "5gram bllip-sm 1111 natural-stories 6 3113"
[1] "5gram bllip-xs 1111 bnc-brown 6 1693"
[1] "5gram bllip-xs 1111 dundee 6 24566"
[1] "5gram bllip-xs 1111 natural-stories 6 3099"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 6 1680"
[1] "gpt2 bllip-lg 1587139950 dundee 6 24564"
[1] "gpt2 bllip-lg 1587139950 natural-stories 6 3083"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 6 1702"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 6 24467"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 6 3082"
[1] "gpt2 bllip-md 1586986276 bnc-brown 6 1710"
[1] "gpt2 bllip-md 1586986276 dundee 6 24555"
[1] "gpt2 bllip-md 1586986276 natural-stories 6 3111"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 6 24579"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 6 3109"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 6 1695"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 6 24582"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 6 3083"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 6 1685"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 6 24606"
[1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 6 3085"

|========================================================================                                                                                                                                                  | 33% ~4 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 6 1678"

|===========================================================================                                                                                                                                               | 34% ~4 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 6 24548"

|=============================================================================                                                                                                                                             | 36% ~4 s remaining     [1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 6 3119"

|================================================================================                                                                                                                                          | 37% ~4 s remaining     [1] "rnng bllip-lg 7245 bnc-brown 6 1695"

|==================================================================================                                                                                                                                        | 38% ~4 s remaining     [1] "rnng bllip-lg 7245 dundee 6 24593"

|=====================================================================================                                                                                                                                     | 39% ~4 s remaining     [1] "rnng bllip-lg 7245 natural-stories 6 3086"

|=======================================================================================                                                                                                                                   | 40% ~4 s remaining     [1] "rnng bllip-md 3602 bnc-brown 6 1683"

|==========================================================================================                                                                                                                                | 41% ~4 s remaining     [1] "rnng bllip-md 3602 dundee 6 24526"

|============================================================================================                                                                                                                              | 43% ~4 s remaining     [1] "rnng bllip-md 3602 natural-stories 6 3062"

|===============================================================================================                                                                                                                           | 44% ~3 s remaining     [1] "rnng bllip-md 44862 bnc-brown 6 1696"

|=================================================================================================                                                                                                                         | 45% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 6 24565"

|====================================================================================================                                                                                                                      | 46% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 6 3098"

|======================================================================================================                                                                                                                    | 47% ~3 s remaining     [1] "rnng bllip-sm 7877 bnc-brown 6 1684"

|=========================================================================================================                                                                                                                 | 48% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 6 24499"

|===========================================================================================================                                                                                                               | 49% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 6 3115"

|==============================================================================================================                                                                                                            | 51% ~3 s remaining     [1] "rnng bllip-sm 64924 bnc-brown 6 1710"

|================================================================================================================                                                                                                          | 52% ~3 s remaining     [1] "rnng bllip-sm 64924 dundee 6 24602"

|===================================================================================================================                                                                                                       | 53% ~3 s remaining     [1] "rnng bllip-sm 64924 natural-stories 6 3093"

|=====================================================================================================================                                                                                                     | 54% ~3 s remaining     [1] "rnng bllip-xs 4301 bnc-brown 6 1675"

|========================================================================================================================                                                                                                  | 55% ~3 s remaining     [1] "rnng bllip-xs 4301 dundee 6 24533"

|==========================================================================================================================                                                                                                | 56% ~3 s remaining     [1] "rnng bllip-xs 4301 natural-stories 6 3051"

|=============================================================================================================================                                                                                             | 57% ~3 s remaining     [1] "rnng bllip-xs 28066 bnc-brown 6 1707"

|===============================================================================================================================                                                                                           | 59% ~3 s remaining     [1] "rnng bllip-xs 28066 dundee 6 24583"

|==================================================================================================================================                                                                                        | 60% ~3 s remaining     [1] "rnng bllip-xs 28066 natural-stories 6 3110"

|====================================================================================================================================                                                                                      | 61% ~3 s remaining     [1] "rnng bllip-xs 28068 bnc-brown 6 1678"

|=======================================================================================================================================                                                                                   | 62% ~3 s remaining     [1] "rnng bllip-xs 28068 dundee 6 24490"

|=========================================================================================================================================                                                                                 | 63% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 6 3110"

|============================================================================================================================================                                                                              | 64% ~2 s remaining     [1] "rnng bllip-xs 51272 bnc-brown 6 1676"

|==============================================================================================================================================                                                                            | 66% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 6 24501"

|=================================================================================================================================================                                                                         | 67% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 6 3080"

|===================================================================================================================================================                                                                       | 68% ~2 s remaining     [1] "vanilla bllip-lg 111 bnc-brown 6 1678"

|======================================================================================================================================================                                                                    | 69% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 6 24526"

|========================================================================================================================================================                                                                  | 70% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 6 3096"

|===========================================================================================================================================================                                                               | 71% ~2 s remaining     [1] "vanilla bllip-md 120 bnc-brown 6 1691"

|=============================================================================================================================================================                                                             | 72% ~2 s remaining     [1] "vanilla bllip-md 120 dundee 6 24583"

|================================================================================================================================================================                                                          | 74% ~2 s remaining     [1] "vanilla bllip-md 120 natural-stories 6 3079"

|==================================================================================================================================================================                                                        | 75% ~2 s remaining     [1] "vanilla bllip-md 607 bnc-brown 6 1699"

|=====================================================================================================================================================================                                                     | 76% ~2 s remaining     [1] "vanilla bllip-md 922 bnc-brown 6 1696"

|=======================================================================================================================================================================                                                   | 77% ~2 s remaining     [1] "vanilla bllip-md 922 dundee 6 24614"

|==========================================================================================================================================================================                                                | 78% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 6 3092"

|============================================================================================================================================================================                                              | 79% ~1 s remaining     [1] "vanilla bllip-sm 111 bnc-brown 6 1695"

|===============================================================================================================================================================================                                           | 80% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 6 24613"

|=================================================================================================================================================================================                                         | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 6 3100"

|====================================================================================================================================================================================                                      | 83% ~1 s remaining     [1] "vanilla bllip-sm 120 bnc-brown 6 1670"

|======================================================================================================================================================================================                                    | 84% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 6 24499"

|=========================================================================================================================================================================================                                 | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 6 3098"

|===========================================================================================================================================================================================                               | 86% ~1 s remaining     [1] "vanilla bllip-sm 922 bnc-brown 6 1667"

|==============================================================================================================================================================================================                            | 87% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 6 24550"

|================================================================================================================================================================================================                          | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 6 3111"

|===================================================================================================================================================================================================                       | 90% ~1 s remaining     [1] "vanilla bllip-xs 111 bnc-brown 6 1700"

|=====================================================================================================================================================================================================                     | 91% ~1 s remaining     [1] "vanilla bllip-xs 111 dundee 6 24474"

|========================================================================================================================================================================================================                  | 92% ~1 s remaining     [1] "vanilla bllip-xs 111 natural-stories 6 3132"

|==========================================================================================================================================================================================================                | 93% ~0 s remaining     [1] "vanilla bllip-xs 120 bnc-brown 6 1667"

|=============================================================================================================================================================================================================             | 94% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 6 24510"

|===============================================================================================================================================================================================================           | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 6 3135"

|==================================================================================================================================================================================================================        | 97% ~0 s remaining     [1] "vanilla bllip-xs 922 bnc-brown 6 1700"

|====================================================================================================================================================================================================================      | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 6 24500"

|=======================================================================================================================================================================================================================   | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 6 3082"

|==========================================================================================================================================================================================================================|100% ~0 s remaining     

[1] "5gram bllip-lg 1111 bnc-brown 7 1689"
[1] "5gram bllip-lg 1111 dundee 7 24606"
[1] "5gram bllip-lg 1111 natural-stories 7 3088"
[1] "5gram bllip-md 1111 bnc-brown 7 1692"
[1] "5gram bllip-md 1111 dundee 7 24585"
[1] "5gram bllip-md 1111 natural-stories 7 3055"
[1] "5gram bllip-sm 1111 bnc-brown 7 1660"
[1] "5gram bllip-sm 1111 dundee 7 24477"
[1] "5gram bllip-sm 1111 natural-stories 7 3079"
[1] "5gram bllip-xs 1111 bnc-brown 7 1715"
[1] "5gram bllip-xs 1111 dundee 7 24540"
[1] "5gram bllip-xs 1111 natural-stories 7 3103"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 7 1692"
[1] "gpt2 bllip-lg 1587139950 dundee 7 24582"
[1] "gpt2 bllip-lg 1587139950 natural-stories 7 3135"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 7 1696"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 7 24547"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 7 3113"
[1] "gpt2 bllip-md 1586986276 bnc-brown 7 1697"
[1] "gpt2 bllip-md 1586986276 dundee 7 24561"
[1] "gpt2 bllip-md 1586986276 natural-stories 7 3096"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 7 24515"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 7 3100"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 7 1698"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 7 24564"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 7 3081"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 7 1692"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 7 24598"
[1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 7 3110"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 7 1704"
[1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 7 24563"
[1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 7 3090"
[1] "rnng bllip-lg 7245 bnc-brown 7 1721"
[1] "rnng bllip-lg 7245 dundee 7 24505"
[1] "rnng bllip-lg 7245 natural-stories 7 3100"
[1] "rnng bllip-md 3602 bnc-brown 7 1688"
[1] "rnng bllip-md 3602 dundee 7 24579"
[1] "rnng bllip-md 3602 natural-stories 7 3086"
[1] "rnng bllip-md 44862 bnc-brown 7 1703"
[1] "rnng bllip-md 44862 dundee 7 24562"

|====================================================================================================                                                                                                                      | 46% ~2 s remaining     [1] "rnng bllip-md 44862 natural-stories 7 3096"

|======================================================================================================                                                                                                                    | 47% ~2 s remaining     [1] "rnng bllip-sm 7877 bnc-brown 7 1696"
[1] "rnng bllip-sm 7877 dundee 7 24519"

|===========================================================================================================                                                                                                               | 49% ~2 s remaining     [1] "rnng bllip-sm 7877 natural-stories 7 3110"

|==============================================================================================================                                                                                                            | 51% ~2 s remaining     [1] "rnng bllip-sm 64924 bnc-brown 7 1665"
[1] "rnng bllip-sm 64924 dundee 7 24475"

|===================================================================================================================                                                                                                       | 53% ~2 s remaining     [1] "rnng bllip-sm 64924 natural-stories 7 3063"

|=====================================================================================================================                                                                                                     | 54% ~2 s remaining     [1] "rnng bllip-xs 4301 bnc-brown 7 1683"
[1] "rnng bllip-xs 4301 dundee 7 24651"

|==========================================================================================================================                                                                                                | 56% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 7 3069"

|=============================================================================================================================                                                                                             | 57% ~2 s remaining     [1] "rnng bllip-xs 28066 bnc-brown 7 1686"

|===============================================================================================================================                                                                                           | 59% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 7 24601"

|==================================================================================================================================                                                                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 7 3044"

|====================================================================================================================================                                                                                      | 61% ~2 s remaining     [1] "rnng bllip-xs 28068 bnc-brown 7 1691"
[1] "rnng bllip-xs 28068 dundee 7 24455"

|=========================================================================================================================================                                                                                 | 63% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 7 3077"

|============================================================================================================================================                                                                              | 64% ~2 s remaining     [1] "rnng bllip-xs 51272 bnc-brown 7 1681"
[1] "rnng bllip-xs 51272 dundee 7 24537"

|=================================================================================================================================================                                                                         | 67% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 7 3088"

|===================================================================================================================================================                                                                       | 68% ~2 s remaining     [1] "vanilla bllip-lg 111 bnc-brown 7 1703"
[1] "vanilla bllip-lg 111 dundee 7 24516"

|========================================================================================================================================================                                                                  | 70% ~1 s remaining     [1] "vanilla bllip-lg 111 natural-stories 7 3108"

|===========================================================================================================================================================                                                               | 71% ~1 s remaining     [1] "vanilla bllip-md 120 bnc-brown 7 1695"

|=============================================================================================================================================================                                                             | 72% ~1 s remaining     [1] "vanilla bllip-md 120 dundee 7 24451"

|================================================================================================================================================================                                                          | 74% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 7 3083"

|==================================================================================================================================================================                                                        | 75% ~1 s remaining     [1] "vanilla bllip-md 607 bnc-brown 7 1715"

|=====================================================================================================================================================================                                                     | 76% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 7 1705"
[1] "vanilla bllip-md 922 dundee 7 24561"

|==========================================================================================================================================================================                                                | 78% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 7 3094"

|============================================================================================================================================================================                                              | 79% ~1 s remaining     [1] "vanilla bllip-sm 111 bnc-brown 7 1702"

|===============================================================================================================================================================================                                           | 80% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 7 24629"

|=================================================================================================================================================================================                                         | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 7 3098"

|====================================================================================================================================================================================                                      | 83% ~1 s remaining     [1] "vanilla bllip-sm 120 bnc-brown 7 1690"

|======================================================================================================================================================================================                                    | 84% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 7 24601"

|=========================================================================================================================================================================================                                 | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 7 3094"

|===========================================================================================================================================================================================                               | 86% ~1 s remaining     [1] "vanilla bllip-sm 922 bnc-brown 7 1703"

|==============================================================================================================================================================================================                            | 87% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 7 24640"

|================================================================================================================================================================================================                          | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 7 3097"

|===================================================================================================================================================================================================                       | 90% ~1 s remaining     [1] "vanilla bllip-xs 111 bnc-brown 7 1699"

|=====================================================================================================================================================================================================                     | 91% ~0 s remaining     [1] "vanilla bllip-xs 111 dundee 7 24572"

|========================================================================================================================================================================================================                  | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 7 3093"

|==========================================================================================================================================================================================================                | 93% ~0 s remaining     [1] "vanilla bllip-xs 120 bnc-brown 7 1665"
[1] "vanilla bllip-xs 120 dundee 7 24529"

|===============================================================================================================================================================================================================           | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 7 3093"

|==================================================================================================================================================================================================================        | 97% ~0 s remaining     [1] "vanilla bllip-xs 922 bnc-brown 7 1696"

|====================================================================================================================================================================================================================      | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 7 24451"

|=======================================================================================================================================================================================================================   | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 7 3118"

|==========================================================================================================================================================================================================================|100% ~0 s remaining     [1] "5gram bllip-lg 1111 bnc-brown 7 1689"
[1] "5gram bllip-lg 1111 dundee 7 24606"
[1] "5gram bllip-lg 1111 natural-stories 7 3088"
[1] "5gram bllip-md 1111 bnc-brown 7 1692"
[1] "5gram bllip-md 1111 dundee 7 24585"
[1] "5gram bllip-md 1111 natural-stories 7 3055"
[1] "5gram bllip-sm 1111 bnc-brown 7 1660"
[1] "5gram bllip-sm 1111 dundee 7 24477"
[1] "5gram bllip-sm 1111 natural-stories 7 3079"
[1] "5gram bllip-xs 1111 bnc-brown 7 1715"
[1] "5gram bllip-xs 1111 dundee 7 24540"
[1] "5gram bllip-xs 1111 natural-stories 7 3103"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 7 1692"
[1] "gpt2 bllip-lg 1587139950 dundee 7 24582"
[1] "gpt2 bllip-lg 1587139950 natural-stories 7 3135"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 7 1696"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 7 24547"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 7 3113"
[1] "gpt2 bllip-md 1586986276 bnc-brown 7 1697"
[1] "gpt2 bllip-md 1586986276 dundee 7 24561"
[1] "gpt2 bllip-md 1586986276 natural-stories 7 3096"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 7 24515"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 7 3100"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 7 1698"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 7 24564"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 7 3081"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 7 1692"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 7 24598"
[1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 7 3110"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 7 1704"
[1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 7 24563"
[1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 7 3090"
[1] "rnng bllip-lg 7245 bnc-brown 7 1721"
[1] "rnng bllip-lg 7245 dundee 7 24505"
[1] "rnng bllip-lg 7245 natural-stories 7 3100"
[1] "rnng bllip-md 3602 bnc-brown 7 1688"
[1] "rnng bllip-md 3602 dundee 7 24579"

|============================================================================================                                                                                                                              | 43% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 7 3086"

|===============================================================================================                                                                                                                           | 44% ~3 s remaining     [1] "rnng bllip-md 44862 bnc-brown 7 1703"

|=================================================================================================                                                                                                                         | 45% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 7 24562"

|====================================================================================================                                                                                                                      | 46% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 7 3096"

|======================================================================================================                                                                                                                    | 47% ~3 s remaining     [1] "rnng bllip-sm 7877 bnc-brown 7 1696"

|=========================================================================================================                                                                                                                 | 48% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 7 24519"

|===========================================================================================================                                                                                                               | 49% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 7 3110"

|==============================================================================================================                                                                                                            | 51% ~3 s remaining     [1] "rnng bllip-sm 64924 bnc-brown 7 1665"

|================================================================================================================                                                                                                          | 52% ~3 s remaining     [1] "rnng bllip-sm 64924 dundee 7 24475"

|===================================================================================================================                                                                                                       | 53% ~3 s remaining     [1] "rnng bllip-sm 64924 natural-stories 7 3063"

|=====================================================================================================================                                                                                                     | 54% ~3 s remaining     [1] "rnng bllip-xs 4301 bnc-brown 7 1683"

|========================================================================================================================                                                                                                  | 55% ~2 s remaining     [1] "rnng bllip-xs 4301 dundee 7 24651"

|==========================================================================================================================                                                                                                | 56% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 7 3069"

|=============================================================================================================================                                                                                             | 57% ~2 s remaining     [1] "rnng bllip-xs 28066 bnc-brown 7 1686"

|===============================================================================================================================                                                                                           | 59% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 7 24601"

|==================================================================================================================================                                                                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 7 3044"

|====================================================================================================================================                                                                                      | 61% ~2 s remaining     [1] "rnng bllip-xs 28068 bnc-brown 7 1691"

|=======================================================================================================================================                                                                                   | 62% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 7 24455"

|=========================================================================================================================================                                                                                 | 63% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 7 3077"

|============================================================================================================================================                                                                              | 64% ~2 s remaining     [1] "rnng bllip-xs 51272 bnc-brown 7 1681"

|==============================================================================================================================================                                                                            | 66% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 7 24537"

|=================================================================================================================================================                                                                         | 67% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 7 3088"

|===================================================================================================================================================                                                                       | 68% ~2 s remaining     [1] "vanilla bllip-lg 111 bnc-brown 7 1703"

|======================================================================================================================================================                                                                    | 69% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 7 24516"

|========================================================================================================================================================                                                                  | 70% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 7 3108"

|===========================================================================================================================================================                                                               | 71% ~2 s remaining     [1] "vanilla bllip-md 120 bnc-brown 7 1695"

|=============================================================================================================================================================                                                             | 72% ~2 s remaining     [1] "vanilla bllip-md 120 dundee 7 24451"

|================================================================================================================================================================                                                          | 74% ~2 s remaining     [1] "vanilla bllip-md 120 natural-stories 7 3083"

|==================================================================================================================================================================                                                        | 75% ~1 s remaining     [1] "vanilla bllip-md 607 bnc-brown 7 1715"

|=====================================================================================================================================================================                                                     | 76% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 7 1705"

|=======================================================================================================================================================================                                                   | 77% ~1 s remaining     [1] "vanilla bllip-md 922 dundee 7 24561"

|==========================================================================================================================================================================                                                | 78% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 7 3094"

|============================================================================================================================================================================                                              | 79% ~1 s remaining     [1] "vanilla bllip-sm 111 bnc-brown 7 1702"

|===============================================================================================================================================================================                                           | 80% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 7 24629"

|=================================================================================================================================================================================                                         | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 7 3098"

|====================================================================================================================================================================================                                      | 83% ~1 s remaining     [1] "vanilla bllip-sm 120 bnc-brown 7 1690"

|======================================================================================================================================================================================                                    | 84% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 7 24601"

|=========================================================================================================================================================================================                                 | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 7 3094"

|===========================================================================================================================================================================================                               | 86% ~1 s remaining     [1] "vanilla bllip-sm 922 bnc-brown 7 1703"

|==============================================================================================================================================================================================                            | 87% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 7 24640"

|================================================================================================================================================================================================                          | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 7 3097"

|===================================================================================================================================================================================================                       | 90% ~1 s remaining     [1] "vanilla bllip-xs 111 bnc-brown 7 1699"

|=====================================================================================================================================================================================================                     | 91% ~1 s remaining     [1] "vanilla bllip-xs 111 dundee 7 24572"

|========================================================================================================================================================================================================                  | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 7 3093"

|==========================================================================================================================================================================================================                | 93% ~0 s remaining     [1] "vanilla bllip-xs 120 bnc-brown 7 1665"

|=============================================================================================================================================================================================================             | 94% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 7 24529"

|===============================================================================================================================================================================================================           | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 7 3093"

|==================================================================================================================================================================================================================        | 97% ~0 s remaining     [1] "vanilla bllip-xs 922 bnc-brown 7 1696"

|====================================================================================================================================================================================================================      | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 7 24451"

|=======================================================================================================================================================================================================================   | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 7 3118"

|==========================================================================================================================================================================================================================|100% ~0 s remaining     

[1] "5gram bllip-lg 1111 bnc-brown 8 1695"
[1] "5gram bllip-lg 1111 dundee 8 24543"
[1] "5gram bllip-lg 1111 natural-stories 8 3111"
[1] "5gram bllip-md 1111 bnc-brown 8 1709"
[1] "5gram bllip-md 1111 dundee 8 24584"
[1] "5gram bllip-md 1111 natural-stories 8 3110"
[1] "5gram bllip-sm 1111 bnc-brown 8 1714"
[1] "5gram bllip-sm 1111 dundee 8 24525"
[1] "5gram bllip-sm 1111 natural-stories 8 3079"
[1] "5gram bllip-xs 1111 bnc-brown 8 1698"
[1] "5gram bllip-xs 1111 dundee 8 24522"
[1] "5gram bllip-xs 1111 natural-stories 8 3076"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 8 1717"
[1] "gpt2 bllip-lg 1587139950 dundee 8 24574"
[1] "gpt2 bllip-lg 1587139950 natural-stories 8 3069"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 8 1686"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 8 24478"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 8 3093"
[1] "gpt2 bllip-md 1586986276 bnc-brown 8 1689"
[1] "gpt2 bllip-md 1586986276 dundee 8 24582"
[1] "gpt2 bllip-md 1586986276 natural-stories 8 3072"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 8 24451"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 8 3134"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 8 1699"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 8 24572"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 8 3121"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 8 1702"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 8 24506"
[1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 8 3076"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 8 1711"
[1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 8 24545"
[1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 8 3054"
[1] "rnng bllip-lg 7245 bnc-brown 8 1690"
[1] "rnng bllip-lg 7245 dundee 8 24553"
[1] "rnng bllip-lg 7245 natural-stories 8 3098"
[1] "rnng bllip-md 3602 bnc-brown 8 1699"
[1] "rnng bllip-md 3602 dundee 8 24516"
[1] "rnng bllip-md 3602 natural-stories 8 3109"
[1] "rnng bllip-md 44862 bnc-brown 8 1677"
[1] "rnng bllip-md 44862 dundee 8 24625"

|====================================================================================================                                                                                                                      | 46% ~2 s remaining     [1] "rnng bllip-md 44862 natural-stories 8 3132"

|======================================================================================================                                                                                                                    | 47% ~2 s remaining     [1] "rnng bllip-sm 7877 bnc-brown 8 1690"

|=========================================================================================================                                                                                                                 | 48% ~2 s remaining     [1] "rnng bllip-sm 7877 dundee 8 24540"

|===========================================================================================================                                                                                                               | 49% ~2 s remaining     [1] "rnng bllip-sm 7877 natural-stories 8 3094"

|==============================================================================================================                                                                                                            | 51% ~2 s remaining     [1] "rnng bllip-sm 64924 bnc-brown 8 1693"

|================================================================================================================                                                                                                          | 52% ~2 s remaining     [1] "rnng bllip-sm 64924 dundee 8 24506"

|===================================================================================================================                                                                                                       | 53% ~2 s remaining     [1] "rnng bllip-sm 64924 natural-stories 8 3090"

|=====================================================================================================================                                                                                                     | 54% ~2 s remaining     [1] "rnng bllip-xs 4301 bnc-brown 8 1719"

|========================================================================================================================                                                                                                  | 55% ~2 s remaining     [1] "rnng bllip-xs 4301 dundee 8 24563"

|==========================================================================================================================                                                                                                | 56% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 8 3105"

|=============================================================================================================================                                                                                             | 57% ~2 s remaining     [1] "rnng bllip-xs 28066 bnc-brown 8 1698"

|===============================================================================================================================                                                                                           | 59% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 8 24582"

|==================================================================================================================================                                                                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 8 3097"

|====================================================================================================================================                                                                                      | 61% ~2 s remaining     [1] "rnng bllip-xs 28068 bnc-brown 8 1696"

|=======================================================================================================================================                                                                                   | 62% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 8 24557"

|=========================================================================================================================================                                                                                 | 63% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 8 3070"

|============================================================================================================================================                                                                              | 64% ~2 s remaining     [1] "rnng bllip-xs 51272 bnc-brown 8 1699"

|==============================================================================================================================================                                                                            | 66% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 8 24546"

|=================================================================================================================================================                                                                         | 67% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 8 3107"

|===================================================================================================================================================                                                                       | 68% ~2 s remaining     [1] "vanilla bllip-lg 111 bnc-brown 8 1684"

|======================================================================================================================================================                                                                    | 69% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 8 24524"

|========================================================================================================================================================                                                                  | 70% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 8 3085"

|===========================================================================================================================================================                                                               | 71% ~2 s remaining     [1] "vanilla bllip-md 120 bnc-brown 8 1704"

|=============================================================================================================================================================                                                             | 72% ~2 s remaining     [1] "vanilla bllip-md 120 dundee 8 24576"

|================================================================================================================================================================                                                          | 74% ~1 s remaining     [1] "vanilla bllip-md 120 natural-stories 8 3079"

|==================================================================================================================================================================                                                        | 75% ~1 s remaining     [1] "vanilla bllip-md 607 bnc-brown 8 1674"

|=====================================================================================================================================================================                                                     | 76% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 8 1691"

|=======================================================================================================================================================================                                                   | 77% ~1 s remaining     [1] "vanilla bllip-md 922 dundee 8 24506"

|==========================================================================================================================================================================                                                | 78% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 8 3123"

|============================================================================================================================================================================                                              | 79% ~1 s remaining     [1] "vanilla bllip-sm 111 bnc-brown 8 1700"
[1] "vanilla bllip-sm 111 dundee 8 24536"

|=================================================================================================================================================================================                                         | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 8 3083"

|====================================================================================================================================================================================                                      | 83% ~1 s remaining     [1] "vanilla bllip-sm 120 bnc-brown 8 1691"

|======================================================================================================================================================================================                                    | 84% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 8 24505"

|=========================================================================================================================================================================================                                 | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 8 3108"

|===========================================================================================================================================================================================                               | 86% ~1 s remaining     [1] "vanilla bllip-sm 922 bnc-brown 8 1706"

|==============================================================================================================================================================================================                            | 87% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 8 24565"

|================================================================================================================================================================================================                          | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 8 3093"

|===================================================================================================================================================================================================                       | 90% ~1 s remaining     [1] "vanilla bllip-xs 111 bnc-brown 8 1678"

|=====================================================================================================================================================================================================                     | 91% ~1 s remaining     [1] "vanilla bllip-xs 111 dundee 8 24569"

|========================================================================================================================================================================================================                  | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 8 3107"

|==========================================================================================================================================================================================================                | 93% ~0 s remaining     [1] "vanilla bllip-xs 120 bnc-brown 8 1690"
[1] "vanilla bllip-xs 120 dundee 8 24520"

|===============================================================================================================================================================================================================           | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 8 3124"

|==================================================================================================================================================================================================================        | 97% ~0 s remaining     [1] "vanilla bllip-xs 922 bnc-brown 8 1689"

|====================================================================================================================================================================================================================      | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 8 24598"

|=======================================================================================================================================================================================================================   | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 8 3100"

|==========================================================================================================================================================================================================================|100% ~0 s remaining     [1] "5gram bllip-lg 1111 bnc-brown 8 1695"
[1] "5gram bllip-lg 1111 dundee 8 24543"
[1] "5gram bllip-lg 1111 natural-stories 8 3111"
[1] "5gram bllip-md 1111 bnc-brown 8 1709"
[1] "5gram bllip-md 1111 dundee 8 24584"
[1] "5gram bllip-md 1111 natural-stories 8 3110"
[1] "5gram bllip-sm 1111 bnc-brown 8 1714"
[1] "5gram bllip-sm 1111 dundee 8 24525"
[1] "5gram bllip-sm 1111 natural-stories 8 3079"
[1] "5gram bllip-xs 1111 bnc-brown 8 1698"
[1] "5gram bllip-xs 1111 dundee 8 24522"
[1] "5gram bllip-xs 1111 natural-stories 8 3076"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 8 1717"
[1] "gpt2 bllip-lg 1587139950 dundee 8 24574"
[1] "gpt2 bllip-lg 1587139950 natural-stories 8 3069"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 8 1686"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 8 24478"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 8 3093"
[1] "gpt2 bllip-md 1586986276 bnc-brown 8 1689"
[1] "gpt2 bllip-md 1586986276 dundee 8 24582"
[1] "gpt2 bllip-md 1586986276 natural-stories 8 3072"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 8 24451"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 8 3134"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 8 1699"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 8 24572"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 8 3121"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 8 1702"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 8 24506"
[1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 8 3076"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 8 1711"
[1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 8 24545"
[1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 8 3054"
[1] "rnng bllip-lg 7245 bnc-brown 8 1690"
[1] "rnng bllip-lg 7245 dundee 8 24553"
[1] "rnng bllip-lg 7245 natural-stories 8 3098"
[1] "rnng bllip-md 3602 bnc-brown 8 1699"
[1] "rnng bllip-md 3602 dundee 8 24516"

|============================================================================================                                                                                                                              | 43% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 8 3109"

|===============================================================================================                                                                                                                           | 44% ~3 s remaining     [1] "rnng bllip-md 44862 bnc-brown 8 1677"

|=================================================================================================                                                                                                                         | 45% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 8 24625"

|====================================================================================================                                                                                                                      | 46% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 8 3132"

|======================================================================================================                                                                                                                    | 47% ~3 s remaining     [1] "rnng bllip-sm 7877 bnc-brown 8 1690"

|=========================================================================================================                                                                                                                 | 48% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 8 24540"

|===========================================================================================================                                                                                                               | 49% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 8 3094"

|==============================================================================================================                                                                                                            | 51% ~3 s remaining     [1] "rnng bllip-sm 64924 bnc-brown 8 1693"

|================================================================================================================                                                                                                          | 52% ~3 s remaining     [1] "rnng bllip-sm 64924 dundee 8 24506"

|===================================================================================================================                                                                                                       | 53% ~3 s remaining     [1] "rnng bllip-sm 64924 natural-stories 8 3090"

|=====================================================================================================================                                                                                                     | 54% ~2 s remaining     [1] "rnng bllip-xs 4301 bnc-brown 8 1719"

|========================================================================================================================                                                                                                  | 55% ~2 s remaining     [1] "rnng bllip-xs 4301 dundee 8 24563"

|==========================================================================================================================                                                                                                | 56% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 8 3105"

|=============================================================================================================================                                                                                             | 57% ~2 s remaining     [1] "rnng bllip-xs 28066 bnc-brown 8 1698"

|===============================================================================================================================                                                                                           | 59% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 8 24582"

|==================================================================================================================================                                                                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 8 3097"

|====================================================================================================================================                                                                                      | 61% ~2 s remaining     [1] "rnng bllip-xs 28068 bnc-brown 8 1696"

|=======================================================================================================================================                                                                                   | 62% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 8 24557"

|=========================================================================================================================================                                                                                 | 63% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 8 3070"

|============================================================================================================================================                                                                              | 64% ~2 s remaining     [1] "rnng bllip-xs 51272 bnc-brown 8 1699"

|==============================================================================================================================================                                                                            | 66% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 8 24546"

|=================================================================================================================================================                                                                         | 67% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 8 3107"

|===================================================================================================================================================                                                                       | 68% ~2 s remaining     [1] "vanilla bllip-lg 111 bnc-brown 8 1684"

|======================================================================================================================================================                                                                    | 69% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 8 24524"

|========================================================================================================================================================                                                                  | 70% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 8 3085"

|===========================================================================================================================================================                                                               | 71% ~2 s remaining     [1] "vanilla bllip-md 120 bnc-brown 8 1704"

|=============================================================================================================================================================                                                             | 72% ~2 s remaining     [1] "vanilla bllip-md 120 dundee 8 24576"

|================================================================================================================================================================                                                          | 74% ~2 s remaining     [1] "vanilla bllip-md 120 natural-stories 8 3079"

|==================================================================================================================================================================                                                        | 75% ~2 s remaining     [1] "vanilla bllip-md 607 bnc-brown 8 1674"

|=====================================================================================================================================================================                                                     | 76% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 8 1691"

|=======================================================================================================================================================================                                                   | 77% ~1 s remaining     [1] "vanilla bllip-md 922 dundee 8 24506"

|==========================================================================================================================================================================                                                | 78% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 8 3123"

|============================================================================================================================================================================                                              | 79% ~1 s remaining     [1] "vanilla bllip-sm 111 bnc-brown 8 1700"

|===============================================================================================================================================================================                                           | 80% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 8 24536"

|=================================================================================================================================================================================                                         | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 8 3083"

|====================================================================================================================================================================================                                      | 83% ~1 s remaining     [1] "vanilla bllip-sm 120 bnc-brown 8 1691"

|======================================================================================================================================================================================                                    | 84% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 8 24505"

|=========================================================================================================================================================================================                                 | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 8 3108"

|===========================================================================================================================================================================================                               | 86% ~1 s remaining     [1] "vanilla bllip-sm 922 bnc-brown 8 1706"

|==============================================================================================================================================================================================                            | 87% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 8 24565"

|================================================================================================================================================================================================                          | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 8 3093"

|===================================================================================================================================================================================================                       | 90% ~1 s remaining     [1] "vanilla bllip-xs 111 bnc-brown 8 1678"

|=====================================================================================================================================================================================================                     | 91% ~1 s remaining     [1] "vanilla bllip-xs 111 dundee 8 24569"

|========================================================================================================================================================================================================                  | 92% ~1 s remaining     [1] "vanilla bllip-xs 111 natural-stories 8 3107"

|==========================================================================================================================================================================================================                | 93% ~0 s remaining     [1] "vanilla bllip-xs 120 bnc-brown 8 1690"

|=============================================================================================================================================================================================================             | 94% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 8 24520"

|===============================================================================================================================================================================================================           | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 8 3124"

|==================================================================================================================================================================================================================        | 97% ~0 s remaining     [1] "vanilla bllip-xs 922 bnc-brown 8 1689"

|====================================================================================================================================================================================================================      | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 8 24598"

|=======================================================================================================================================================================================================================   | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 8 3100"

|==========================================================================================================================================================================================================================|100% ~0 s remaining     

[1] "5gram bllip-lg 1111 bnc-brown 9 1687"
[1] "5gram bllip-lg 1111 dundee 9 24569"
[1] "5gram bllip-lg 1111 natural-stories 9 3082"
[1] "5gram bllip-md 1111 bnc-brown 9 1689"
[1] "5gram bllip-md 1111 dundee 9 24471"
[1] "5gram bllip-md 1111 natural-stories 9 3130"
[1] "5gram bllip-sm 1111 bnc-brown 9 1717"
[1] "5gram bllip-sm 1111 dundee 9 24527"
[1] "5gram bllip-sm 1111 natural-stories 9 3066"
[1] "5gram bllip-xs 1111 bnc-brown 9 1716"
[1] "5gram bllip-xs 1111 dundee 9 24483"
[1] "5gram bllip-xs 1111 natural-stories 9 3051"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 9 1713"
[1] "gpt2 bllip-lg 1587139950 dundee 9 24520"
[1] "gpt2 bllip-lg 1587139950 natural-stories 9 3089"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 9 1690"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 9 24509"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 9 3106"
[1] "gpt2 bllip-md 1586986276 bnc-brown 9 1704"
[1] "gpt2 bllip-md 1586986276 dundee 9 24501"
[1] "gpt2 bllip-md 1586986276 natural-stories 9 3095"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 9 24581"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 9 3101"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 9 1676"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 9 24588"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 9 3102"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 9 1708"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 9 24596"
[1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 9 3104"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 9 1689"
[1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 9 24487"
[1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 9 3111"
[1] "rnng bllip-lg 7245 bnc-brown 9 1699"
[1] "rnng bllip-lg 7245 dundee 9 24547"
[1] "rnng bllip-lg 7245 natural-stories 9 3107"
[1] "rnng bllip-md 3602 bnc-brown 9 1701"
[1] "rnng bllip-md 3602 dundee 9 24566"
[1] "rnng bllip-md 3602 natural-stories 9 3093"
[1] "rnng bllip-md 44862 bnc-brown 9 1687"
[1] "rnng bllip-md 44862 dundee 9 24523"

|====================================================================================================                                                                                                                      | 46% ~2 s remaining     [1] "rnng bllip-md 44862 natural-stories 9 3082"

|======================================================================================================                                                                                                                    | 47% ~2 s remaining     [1] "rnng bllip-sm 7877 bnc-brown 9 1712"
[1] "rnng bllip-sm 7877 dundee 9 24643"

|===========================================================================================================                                                                                                               | 49% ~2 s remaining     [1] "rnng bllip-sm 7877 natural-stories 9 3100"

|==============================================================================================================                                                                                                            | 51% ~2 s remaining     [1] "rnng bllip-sm 64924 bnc-brown 9 1699"
[1] "rnng bllip-sm 64924 dundee 9 24569"

|===================================================================================================================                                                                                                       | 53% ~2 s remaining     [1] "rnng bllip-sm 64924 natural-stories 9 3102"

|=====================================================================================================================                                                                                                     | 54% ~2 s remaining     [1] "rnng bllip-xs 4301 bnc-brown 9 1691"

|========================================================================================================================                                                                                                  | 55% ~2 s remaining     [1] "rnng bllip-xs 4301 dundee 9 24624"

|==========================================================================================================================                                                                                                | 56% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 9 3076"

|=============================================================================================================================                                                                                             | 57% ~2 s remaining     [1] "rnng bllip-xs 28066 bnc-brown 9 1660"

|===============================================================================================================================                                                                                           | 59% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 9 24526"

|==================================================================================================================================                                                                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 9 3107"

|====================================================================================================================================                                                                                      | 61% ~2 s remaining     [1] "rnng bllip-xs 28068 bnc-brown 9 1697"

|=======================================================================================================================================                                                                                   | 62% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 9 24553"

|=========================================================================================================================================                                                                                 | 63% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 9 3105"

|============================================================================================================================================                                                                              | 64% ~2 s remaining     [1] "rnng bllip-xs 51272 bnc-brown 9 1684"

|==============================================================================================================================================                                                                            | 66% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 9 24509"

|=================================================================================================================================================                                                                         | 67% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 9 3091"

|===================================================================================================================================================                                                                       | 68% ~2 s remaining     [1] "vanilla bllip-lg 111 bnc-brown 9 1679"

|======================================================================================================================================================                                                                    | 69% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 9 24641"

|========================================================================================================================================================                                                                  | 70% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 9 3113"

|===========================================================================================================================================================                                                               | 71% ~2 s remaining     [1] "vanilla bllip-md 120 bnc-brown 9 1692"

|=============================================================================================================================================================                                                             | 72% ~2 s remaining     [1] "vanilla bllip-md 120 dundee 9 24487"

|================================================================================================================================================================                                                          | 74% ~2 s remaining     [1] "vanilla bllip-md 120 natural-stories 9 3117"

|==================================================================================================================================================================                                                        | 75% ~2 s remaining     [1] "vanilla bllip-md 607 bnc-brown 9 1686"

|=====================================================================================================================================================================                                                     | 76% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 9 1685"

|=======================================================================================================================================================================                                                   | 77% ~1 s remaining     [1] "vanilla bllip-md 922 dundee 9 24529"

|==========================================================================================================================================================================                                                | 78% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 9 3088"

|============================================================================================================================================================================                                              | 79% ~1 s remaining     [1] "vanilla bllip-sm 111 bnc-brown 9 1680"
[1] "vanilla bllip-sm 111 dundee 9 24517"

|=================================================================================================================================================================================                                         | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 9 3121"

|====================================================================================================================================================================================                                      | 83% ~1 s remaining     [1] "vanilla bllip-sm 120 bnc-brown 9 1703"

|======================================================================================================================================================================================                                    | 84% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 9 24580"

|=========================================================================================================================================================================================                                 | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 9 3094"

|===========================================================================================================================================================================================                               | 86% ~1 s remaining     [1] "vanilla bllip-sm 922 bnc-brown 9 1687"

|==============================================================================================================================================================================================                            | 87% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 9 24500"

|================================================================================================================================================================================================                          | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 9 3061"

|===================================================================================================================================================================================================                       | 90% ~1 s remaining     [1] "vanilla bllip-xs 111 bnc-brown 9 1696"
[1] "vanilla bllip-xs 111 dundee 9 24568"

|========================================================================================================================================================================================================                  | 92% ~1 s remaining     [1] "vanilla bllip-xs 111 natural-stories 9 3084"

|==========================================================================================================================================================================================================                | 93% ~0 s remaining     [1] "vanilla bllip-xs 120 bnc-brown 9 1683"
[1] "vanilla bllip-xs 120 dundee 9 24509"

|===============================================================================================================================================================================================================           | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 9 3092"

|==================================================================================================================================================================================================================        | 97% ~0 s remaining     [1] "vanilla bllip-xs 922 bnc-brown 9 1688"
[1] "vanilla bllip-xs 922 dundee 9 24657"

|=======================================================================================================================================================================================================================   | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 9 3108"

|==========================================================================================================================================================================================================================|100% ~0 s remaining     [1] "5gram bllip-lg 1111 bnc-brown 9 1687"
[1] "5gram bllip-lg 1111 dundee 9 24569"
[1] "5gram bllip-lg 1111 natural-stories 9 3082"
[1] "5gram bllip-md 1111 bnc-brown 9 1689"
[1] "5gram bllip-md 1111 dundee 9 24471"
[1] "5gram bllip-md 1111 natural-stories 9 3130"
[1] "5gram bllip-sm 1111 bnc-brown 9 1717"
[1] "5gram bllip-sm 1111 dundee 9 24527"
[1] "5gram bllip-sm 1111 natural-stories 9 3066"
[1] "5gram bllip-xs 1111 bnc-brown 9 1716"
[1] "5gram bllip-xs 1111 dundee 9 24483"
[1] "5gram bllip-xs 1111 natural-stories 9 3051"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 9 1713"
[1] "gpt2 bllip-lg 1587139950 dundee 9 24520"
[1] "gpt2 bllip-lg 1587139950 natural-stories 9 3089"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 9 1690"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 9 24509"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 9 3106"
[1] "gpt2 bllip-md 1586986276 bnc-brown 9 1704"
[1] "gpt2 bllip-md 1586986276 dundee 9 24501"
[1] "gpt2 bllip-md 1586986276 natural-stories 9 3095"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 9 24581"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 9 3101"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 9 1676"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 9 24588"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 9 3102"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 9 1708"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 9 24596"
[1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 9 3104"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 9 1689"
[1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 9 24487"
[1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 9 3111"
[1] "rnng bllip-lg 7245 bnc-brown 9 1699"
[1] "rnng bllip-lg 7245 dundee 9 24547"
[1] "rnng bllip-lg 7245 natural-stories 9 3107"
[1] "rnng bllip-md 3602 bnc-brown 9 1701"

|==========================================================================================                                                                                                                                | 41% ~3 s remaining     [1] "rnng bllip-md 3602 dundee 9 24566"

|============================================================================================                                                                                                                              | 43% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 9 3093"

|===============================================================================================                                                                                                                           | 44% ~3 s remaining     [1] "rnng bllip-md 44862 bnc-brown 9 1687"

|=================================================================================================                                                                                                                         | 45% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 9 24523"

|====================================================================================================                                                                                                                      | 46% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 9 3082"

|======================================================================================================                                                                                                                    | 47% ~3 s remaining     [1] "rnng bllip-sm 7877 bnc-brown 9 1712"

|=========================================================================================================                                                                                                                 | 48% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 9 24643"

|===========================================================================================================                                                                                                               | 49% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 9 3100"

|==============================================================================================================                                                                                                            | 51% ~3 s remaining     [1] "rnng bllip-sm 64924 bnc-brown 9 1699"

|================================================================================================================                                                                                                          | 52% ~3 s remaining     [1] "rnng bllip-sm 64924 dundee 9 24569"

|===================================================================================================================                                                                                                       | 53% ~3 s remaining     [1] "rnng bllip-sm 64924 natural-stories 9 3102"

|=====================================================================================================================                                                                                                     | 54% ~3 s remaining     [1] "rnng bllip-xs 4301 bnc-brown 9 1691"

|========================================================================================================================                                                                                                  | 55% ~3 s remaining     [1] "rnng bllip-xs 4301 dundee 9 24624"

|==========================================================================================================================                                                                                                | 56% ~3 s remaining     [1] "rnng bllip-xs 4301 natural-stories 9 3076"

|=============================================================================================================================                                                                                             | 57% ~2 s remaining     [1] "rnng bllip-xs 28066 bnc-brown 9 1660"

|===============================================================================================================================                                                                                           | 59% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 9 24526"

|==================================================================================================================================                                                                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 9 3107"

|====================================================================================================================================                                                                                      | 61% ~2 s remaining     [1] "rnng bllip-xs 28068 bnc-brown 9 1697"

|=======================================================================================================================================                                                                                   | 62% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 9 24553"

|=========================================================================================================================================                                                                                 | 63% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 9 3105"

|============================================================================================================================================                                                                              | 64% ~2 s remaining     [1] "rnng bllip-xs 51272 bnc-brown 9 1684"

|==============================================================================================================================================                                                                            | 66% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 9 24509"

|=================================================================================================================================================                                                                         | 67% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 9 3091"

|===================================================================================================================================================                                                                       | 68% ~2 s remaining     [1] "vanilla bllip-lg 111 bnc-brown 9 1679"

|======================================================================================================================================================                                                                    | 69% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 9 24641"

|========================================================================================================================================================                                                                  | 70% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 9 3113"

|===========================================================================================================================================================                                                               | 71% ~2 s remaining     [1] "vanilla bllip-md 120 bnc-brown 9 1692"

|=============================================================================================================================================================                                                             | 72% ~2 s remaining     [1] "vanilla bllip-md 120 dundee 9 24487"

|================================================================================================================================================================                                                          | 74% ~2 s remaining     [1] "vanilla bllip-md 120 natural-stories 9 3117"

|==================================================================================================================================================================                                                        | 75% ~2 s remaining     [1] "vanilla bllip-md 607 bnc-brown 9 1686"

|=====================================================================================================================================================================                                                     | 76% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 9 1685"

|=======================================================================================================================================================================                                                   | 77% ~1 s remaining     [1] "vanilla bllip-md 922 dundee 9 24529"

|==========================================================================================================================================================================                                                | 78% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 9 3088"

|============================================================================================================================================================================                                              | 79% ~1 s remaining     [1] "vanilla bllip-sm 111 bnc-brown 9 1680"

|===============================================================================================================================================================================                                           | 80% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 9 24517"

|=================================================================================================================================================================================                                         | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 9 3121"

|====================================================================================================================================================================================                                      | 83% ~1 s remaining     [1] "vanilla bllip-sm 120 bnc-brown 9 1703"

|======================================================================================================================================================================================                                    | 84% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 9 24580"

|=========================================================================================================================================================================================                                 | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 9 3094"

|===========================================================================================================================================================================================                               | 86% ~1 s remaining     [1] "vanilla bllip-sm 922 bnc-brown 9 1687"

|==============================================================================================================================================================================================                            | 87% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 9 24500"

|================================================================================================================================================================================================                          | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 9 3061"

|===================================================================================================================================================================================================                       | 90% ~1 s remaining     [1] "vanilla bllip-xs 111 bnc-brown 9 1696"

|=====================================================================================================================================================================================================                     | 91% ~1 s remaining     [1] "vanilla bllip-xs 111 dundee 9 24568"

|========================================================================================================================================================================================================                  | 92% ~1 s remaining     [1] "vanilla bllip-xs 111 natural-stories 9 3084"

|==========================================================================================================================================================================================================                | 93% ~0 s remaining     [1] "vanilla bllip-xs 120 bnc-brown 9 1683"

|=============================================================================================================================================================================================================             | 94% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 9 24509"

|===============================================================================================================================================================================================================           | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 9 3092"

|==================================================================================================================================================================================================================        | 97% ~0 s remaining     [1] "vanilla bllip-xs 922 bnc-brown 9 1688"

|====================================================================================================================================================================================================================      | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 9 24657"

|=======================================================================================================================================================================================================================   | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 9 3108"

|==========================================================================================================================================================================================================================|100% ~0 s remaining     

[1] "5gram bllip-lg 1111 bnc-brown 10 1672"
[1] "5gram bllip-lg 1111 dundee 10 24542"
[1] "5gram bllip-lg 1111 natural-stories 10 3099"
[1] "5gram bllip-md 1111 bnc-brown 10 1709"
[1] "5gram bllip-md 1111 dundee 10 24544"
[1] "5gram bllip-md 1111 natural-stories 10 3104"
[1] "5gram bllip-sm 1111 bnc-brown 10 1690"
[1] "5gram bllip-sm 1111 dundee 10 24589"
[1] "5gram bllip-sm 1111 natural-stories 10 3097"
[1] "5gram bllip-xs 1111 bnc-brown 10 1700"
[1] "5gram bllip-xs 1111 dundee 10 24496"
[1] "5gram bllip-xs 1111 natural-stories 10 3111"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 10 1705"
[1] "gpt2 bllip-lg 1587139950 dundee 10 24578"
[1] "gpt2 bllip-lg 1587139950 natural-stories 10 3087"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 10 1710"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 10 24541"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 10 3084"
[1] "gpt2 bllip-md 1586986276 bnc-brown 10 1693"
[1] "gpt2 bllip-md 1586986276 dundee 10 24585"
[1] "gpt2 bllip-md 1586986276 natural-stories 10 3091"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 10 24648"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 10 3086"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 10 1687"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 10 24439"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 10 3093"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 10 1689"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 10 24507"
[1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 10 3108"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 10 1687"
[1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 10 24592"
[1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 10 3120"
[1] "rnng bllip-lg 7245 bnc-brown 10 1691"
[1] "rnng bllip-lg 7245 dundee 10 24527"
[1] "rnng bllip-lg 7245 natural-stories 10 3126"
[1] "rnng bllip-md 3602 bnc-brown 10 1694"
[1] "rnng bllip-md 3602 dundee 10 24539"
[1] "rnng bllip-md 3602 natural-stories 10 3096"

|===============================================================================================                                                                                                                           | 44% ~3 s remaining     [1] "rnng bllip-md 44862 bnc-brown 10 1712"

|=================================================================================================                                                                                                                         | 45% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 10 24505"

|====================================================================================================                                                                                                                      | 46% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 10 3091"

|======================================================================================================                                                                                                                    | 47% ~3 s remaining     [1] "rnng bllip-sm 7877 bnc-brown 10 1695"

|=========================================================================================================                                                                                                                 | 48% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 10 24551"

|===========================================================================================================                                                                                                               | 49% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 10 3097"

|==============================================================================================================                                                                                                            | 51% ~3 s remaining     [1] "rnng bllip-sm 64924 bnc-brown 10 1699"
[1] "rnng bllip-sm 64924 dundee 10 24515"

|===================================================================================================================                                                                                                       | 53% ~3 s remaining     [1] "rnng bllip-sm 64924 natural-stories 10 3081"

|=====================================================================================================================                                                                                                     | 54% ~3 s remaining     [1] "rnng bllip-xs 4301 bnc-brown 10 1697"

|========================================================================================================================                                                                                                  | 55% ~3 s remaining     [1] "rnng bllip-xs 4301 dundee 10 24564"

|==========================================================================================================================                                                                                                | 56% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 10 3119"

|=============================================================================================================================                                                                                             | 57% ~2 s remaining     [1] "rnng bllip-xs 28066 bnc-brown 10 1678"

|===============================================================================================================================                                                                                           | 59% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 10 24583"

|==================================================================================================================================                                                                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 10 3117"

|====================================================================================================================================                                                                                      | 61% ~2 s remaining     [1] "rnng bllip-xs 28068 bnc-brown 10 1715"

|=======================================================================================================================================                                                                                   | 62% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 10 24588"

|=========================================================================================================================================                                                                                 | 63% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 10 3113"

|============================================================================================================================================                                                                              | 64% ~2 s remaining     [1] "rnng bllip-xs 51272 bnc-brown 10 1694"

|==============================================================================================================================================                                                                            | 66% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 10 24595"

|=================================================================================================================================================                                                                         | 67% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 10 3099"

|===================================================================================================================================================                                                                       | 68% ~2 s remaining     [1] "vanilla bllip-lg 111 bnc-brown 10 1688"

|======================================================================================================================================================                                                                    | 69% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 10 24585"

|========================================================================================================================================================                                                                  | 70% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 10 3071"

|===========================================================================================================================================================                                                               | 71% ~2 s remaining     [1] "vanilla bllip-md 120 bnc-brown 10 1677"
[1] "vanilla bllip-md 120 dundee 10 24617"

|================================================================================================================================================================                                                          | 74% ~2 s remaining     [1] "vanilla bllip-md 120 natural-stories 10 3147"

|==================================================================================================================================================================                                                        | 75% ~2 s remaining     [1] "vanilla bllip-md 607 bnc-brown 10 1692"

|=====================================================================================================================================================================                                                     | 76% ~2 s remaining     [1] "vanilla bllip-md 922 bnc-brown 10 1697"

|=======================================================================================================================================================================                                                   | 77% ~2 s remaining     [1] "vanilla bllip-md 922 dundee 10 24533"

|==========================================================================================================================================================================                                                | 78% ~2 s remaining     [1] "vanilla bllip-md 922 natural-stories 10 3108"

|============================================================================================================================================================================                                              | 79% ~1 s remaining     [1] "vanilla bllip-sm 111 bnc-brown 10 1674"

|===============================================================================================================================================================================                                           | 80% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 10 24584"

|=================================================================================================================================================================================                                         | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 10 3069"

|====================================================================================================================================================================================                                      | 83% ~1 s remaining     [1] "vanilla bllip-sm 120 bnc-brown 10 1696"
[1] "vanilla bllip-sm 120 dundee 10 24485"

|=========================================================================================================================================================================================                                 | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 10 3095"

|===========================================================================================================================================================================================                               | 86% ~1 s remaining     [1] "vanilla bllip-sm 922 bnc-brown 10 1703"
[1] "vanilla bllip-sm 922 dundee 10 24477"

|================================================================================================================================================================================================                          | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 10 3087"

|===================================================================================================================================================================================================                       | 90% ~1 s remaining     [1] "vanilla bllip-xs 111 bnc-brown 10 1685"

|=====================================================================================================================================================================================================                     | 91% ~1 s remaining     [1] "vanilla bllip-xs 111 dundee 10 24477"

|========================================================================================================================================================================================================                  | 92% ~1 s remaining     [1] "vanilla bllip-xs 111 natural-stories 10 3080"

|==========================================================================================================================================================================================================                | 93% ~0 s remaining     [1] "vanilla bllip-xs 120 bnc-brown 10 1693"

|=============================================================================================================================================================================================================             | 94% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 10 24521"

|===============================================================================================================================================================================================================           | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 10 3089"

|==================================================================================================================================================================================================================        | 97% ~0 s remaining     [1] "vanilla bllip-xs 922 bnc-brown 10 1701"

|====================================================================================================================================================================================================================      | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 10 24485"

|=======================================================================================================================================================================================================================   | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 10 3075"

|==========================================================================================================================================================================================================================|100% ~0 s remaining     [1] "5gram bllip-lg 1111 bnc-brown 10 1672"
[1] "5gram bllip-lg 1111 dundee 10 24542"
[1] "5gram bllip-lg 1111 natural-stories 10 3099"
[1] "5gram bllip-md 1111 bnc-brown 10 1709"
[1] "5gram bllip-md 1111 dundee 10 24544"
[1] "5gram bllip-md 1111 natural-stories 10 3104"
[1] "5gram bllip-sm 1111 bnc-brown 10 1690"
[1] "5gram bllip-sm 1111 dundee 10 24589"
[1] "5gram bllip-sm 1111 natural-stories 10 3097"
[1] "5gram bllip-xs 1111 bnc-brown 10 1700"
[1] "5gram bllip-xs 1111 dundee 10 24496"
[1] "5gram bllip-xs 1111 natural-stories 10 3111"
[1] "gpt2 bllip-lg 1587139950 bnc-brown 10 1705"
[1] "gpt2 bllip-lg 1587139950 dundee 10 24578"
[1] "gpt2 bllip-lg 1587139950 natural-stories 10 3087"
[1] "gpt2 bllip-lg-gptbpe 1611265210 bnc-brown 10 1710"
[1] "gpt2 bllip-lg-gptbpe 1611265210 dundee 10 24541"
[1] "gpt2 bllip-lg-gptbpe 1611265210 natural-stories 10 3084"
[1] "gpt2 bllip-md 1586986276 bnc-brown 10 1693"
[1] "gpt2 bllip-md 1586986276 dundee 10 24585"
[1] "gpt2 bllip-md 1586986276 natural-stories 10 3091"
[1] "gpt2 bllip-md-gptbpe 1111 dundee 10 24648"
[1] "gpt2 bllip-md-gptbpe 1111 natural-stories 10 3086"
[1] "gpt2 bllip-md-gptbpe 1611262494 bnc-brown 10 1687"
[1] "gpt2 bllip-md-gptbpe 1611262494 dundee 10 24439"
[1] "gpt2 bllip-md-gptbpe 1611262494 natural-stories 10 3093"
[1] "gpt2 bllip-sm-gptbpe 1581807578 bnc-brown 10 1689"
[1] "gpt2 bllip-sm-gptbpe 1581807578 dundee 10 24507"
[1] "gpt2 bllip-sm-gptbpe 1581807578 natural-stories 10 3108"
[1] "gpt2 bllip-xs-gptbpe 1581807512 bnc-brown 10 1687"
[1] "gpt2 bllip-xs-gptbpe 1581807512 dundee 10 24592"
[1] "gpt2 bllip-xs-gptbpe 1581807512 natural-stories 10 3120"
[1] "rnng bllip-lg 7245 bnc-brown 10 1691"
[1] "rnng bllip-lg 7245 dundee 10 24527"
[1] "rnng bllip-lg 7245 natural-stories 10 3126"
[1] "rnng bllip-md 3602 bnc-brown 10 1694"
[1] "rnng bllip-md 3602 dundee 10 24539"

|============================================================================================                                                                                                                              | 43% ~3 s remaining     [1] "rnng bllip-md 3602 natural-stories 10 3096"

|===============================================================================================                                                                                                                           | 44% ~3 s remaining     [1] "rnng bllip-md 44862 bnc-brown 10 1712"

|=================================================================================================                                                                                                                         | 45% ~3 s remaining     [1] "rnng bllip-md 44862 dundee 10 24505"

|====================================================================================================                                                                                                                      | 46% ~3 s remaining     [1] "rnng bllip-md 44862 natural-stories 10 3091"

|======================================================================================================                                                                                                                    | 47% ~3 s remaining     [1] "rnng bllip-sm 7877 bnc-brown 10 1695"

|=========================================================================================================                                                                                                                 | 48% ~3 s remaining     [1] "rnng bllip-sm 7877 dundee 10 24551"

|===========================================================================================================                                                                                                               | 49% ~3 s remaining     [1] "rnng bllip-sm 7877 natural-stories 10 3097"

|==============================================================================================================                                                                                                            | 51% ~3 s remaining     [1] "rnng bllip-sm 64924 bnc-brown 10 1699"

|================================================================================================================                                                                                                          | 52% ~2 s remaining     [1] "rnng bllip-sm 64924 dundee 10 24515"

|===================================================================================================================                                                                                                       | 53% ~2 s remaining     [1] "rnng bllip-sm 64924 natural-stories 10 3081"

|=====================================================================================================================                                                                                                     | 54% ~2 s remaining     [1] "rnng bllip-xs 4301 bnc-brown 10 1697"

|========================================================================================================================                                                                                                  | 55% ~2 s remaining     [1] "rnng bllip-xs 4301 dundee 10 24564"

|==========================================================================================================================                                                                                                | 56% ~2 s remaining     [1] "rnng bllip-xs 4301 natural-stories 10 3119"

|=============================================================================================================================                                                                                             | 57% ~2 s remaining     [1] "rnng bllip-xs 28066 bnc-brown 10 1678"

|===============================================================================================================================                                                                                           | 59% ~2 s remaining     [1] "rnng bllip-xs 28066 dundee 10 24583"

|==================================================================================================================================                                                                                        | 60% ~2 s remaining     [1] "rnng bllip-xs 28066 natural-stories 10 3117"

|====================================================================================================================================                                                                                      | 61% ~2 s remaining     [1] "rnng bllip-xs 28068 bnc-brown 10 1715"

|=======================================================================================================================================                                                                                   | 62% ~2 s remaining     [1] "rnng bllip-xs 28068 dundee 10 24588"

|=========================================================================================================================================                                                                                 | 63% ~2 s remaining     [1] "rnng bllip-xs 28068 natural-stories 10 3113"

|============================================================================================================================================                                                                              | 64% ~2 s remaining     [1] "rnng bllip-xs 51272 bnc-brown 10 1694"

|==============================================================================================================================================                                                                            | 66% ~2 s remaining     [1] "rnng bllip-xs 51272 dundee 10 24595"

|=================================================================================================================================================                                                                         | 67% ~2 s remaining     [1] "rnng bllip-xs 51272 natural-stories 10 3099"

|===================================================================================================================================================                                                                       | 68% ~2 s remaining     [1] "vanilla bllip-lg 111 bnc-brown 10 1688"

|======================================================================================================================================================                                                                    | 69% ~2 s remaining     [1] "vanilla bllip-lg 111 dundee 10 24585"

|========================================================================================================================================================                                                                  | 70% ~2 s remaining     [1] "vanilla bllip-lg 111 natural-stories 10 3071"

|===========================================================================================================================================================                                                               | 71% ~2 s remaining     [1] "vanilla bllip-md 120 bnc-brown 10 1677"

|=============================================================================================================================================================                                                             | 72% ~2 s remaining     [1] "vanilla bllip-md 120 dundee 10 24617"

|================================================================================================================================================================                                                          | 74% ~2 s remaining     [1] "vanilla bllip-md 120 natural-stories 10 3147"

|==================================================================================================================================================================                                                        | 75% ~2 s remaining     [1] "vanilla bllip-md 607 bnc-brown 10 1692"

|=====================================================================================================================================================================                                                     | 76% ~1 s remaining     [1] "vanilla bllip-md 922 bnc-brown 10 1697"

|=======================================================================================================================================================================                                                   | 77% ~1 s remaining     [1] "vanilla bllip-md 922 dundee 10 24533"

|==========================================================================================================================================================================                                                | 78% ~1 s remaining     [1] "vanilla bllip-md 922 natural-stories 10 3108"

|============================================================================================================================================================================                                              | 79% ~1 s remaining     [1] "vanilla bllip-sm 111 bnc-brown 10 1674"

|===============================================================================================================================================================================                                           | 80% ~1 s remaining     [1] "vanilla bllip-sm 111 dundee 10 24584"

|=================================================================================================================================================================================                                         | 82% ~1 s remaining     [1] "vanilla bllip-sm 111 natural-stories 10 3069"

|====================================================================================================================================================================================                                      | 83% ~1 s remaining     [1] "vanilla bllip-sm 120 bnc-brown 10 1696"

|======================================================================================================================================================================================                                    | 84% ~1 s remaining     [1] "vanilla bllip-sm 120 dundee 10 24485"

|=========================================================================================================================================================================================                                 | 85% ~1 s remaining     [1] "vanilla bllip-sm 120 natural-stories 10 3095"

|===========================================================================================================================================================================================                               | 86% ~1 s remaining     [1] "vanilla bllip-sm 922 bnc-brown 10 1703"

|==============================================================================================================================================================================================                            | 87% ~1 s remaining     [1] "vanilla bllip-sm 922 dundee 10 24477"

|================================================================================================================================================================================================                          | 89% ~1 s remaining     [1] "vanilla bllip-sm 922 natural-stories 10 3087"

|===================================================================================================================================================================================================                       | 90% ~1 s remaining     [1] "vanilla bllip-xs 111 bnc-brown 10 1685"

|=====================================================================================================================================================================================================                     | 91% ~1 s remaining     [1] "vanilla bllip-xs 111 dundee 10 24477"

|========================================================================================================================================================================================================                  | 92% ~0 s remaining     [1] "vanilla bllip-xs 111 natural-stories 10 3080"

|==========================================================================================================================================================================================================                | 93% ~0 s remaining     [1] "vanilla bllip-xs 120 bnc-brown 10 1693"

|=============================================================================================================================================================================================================             | 94% ~0 s remaining     [1] "vanilla bllip-xs 120 dundee 10 24521"

|===============================================================================================================================================================================================================           | 95% ~0 s remaining     [1] "vanilla bllip-xs 120 natural-stories 10 3089"

|==================================================================================================================================================================================================================        | 97% ~0 s remaining     [1] "vanilla bllip-xs 922 bnc-brown 10 1701"

|====================================================================================================================================================================================================================      | 98% ~0 s remaining     [1] "vanilla bllip-xs 922 dundee 10 24485"

|=======================================================================================================================================================================================================================   | 99% ~0 s remaining     [1] "vanilla bllip-xs 922 natural-stories 10 3075"

|==========================================================================================================================================================================================================================|100% ~0 s remaining     

#write.csv(full_residuals, "../data/analysis_checkpoints/full_residuals.csv")
#write.csv(baseline_residuals, "../data/analysis_checkpoints/baseline_residuals.csv")

model_deltas = log_lik_deltas %>%
  group_by(model, training, seed, corpus) %>% 
  summarise(mean_delta_log_lik = mean(delta_log_lik),
            sem_delta_log_lik = sd(delta_log_lik) / sqrt(length(delta_log_lik)))

write.csv(full_model_results, "../data/analysis_checkpoints/full_model_result.csv")
write.csv(baseline_results, "../data/analysis_checkpoints/baseline_results.csv")
#full_model_results = read.csv("../data/analysis_checkpoints/ffull_model_results.csv")
#baseline_results = read.csv("../data/analysis_checkpoints/fbaseline_resultsb.csv")

metric <- "ΔLogLik"
#metric <- "-ΔMSE"

# # Select the relevant metric.
model_deltas = model_deltas %>%
    # Retrieve the current test metric
    mutate(delta_test_mean = mean_delta_log_lik,
           delta_test_sem = sem_delta_log_lik) %>%
    # mutate(delta_test_mean = mean_delta_mse,
    #        delta_test_sem = sem_delta_mse)
    
    # Remove the raw metrics.
    select(-mean_delta_log_lik, -sem_delta_log_lik,
           #-mean_delta_mse, -sem_delta_mse
           )
model_deltas

# Sanity check: training on train+test data should yield improved performance over training on just training data. (When evaluating on test data.)
 full_baselines = all_data %>%
   group_by(model, training, seed, corpus) %>%
   summarise(baseline_train_all_test_lik = logLik_test(lm(psychometric ~ len + freq + sent_pos, data=.), semi_join(test_data, ., by=c("training", "model", "seed", "corpus")), semi_join(test_data, ., by=c("training", "model", "seed", "corpus"))$psychometric)) %>%
   ungroup()
 full_baselines
 
 full_baselines %>%
   right_join(baselines, by=c("seed", "training", "model", "corpus")) %>%
   mutate(delta=baseline_train_all_test_lik-baseline_test_lik) %>%
   select(-baseline_lik) # %>%
   #select(-baseline_test_lik, -baseline_train_all_test_lik, -baseline_lik, -baseline_test_mse)

Load language model data (SyntaxGym, PPL)

language_model_data = read.csv("../data/model_metadata.csv") %>%
  mutate(model = as.character(model),
         model = if_else(model == "gpt-2", "gpt2", model),
         model = as.factor(model)) %>%
  mutate(train_size = case_when(str_starts(training, "bllip-lg") ~ 42,
                                str_starts(training, "bllip-md") ~ 15,
                                str_starts(training, "bllip-sm") ~ 5,
                                str_starts(training, "bllip-xs") ~ 1),
         
         # Training vocabulary usually covaries with the training corpus.
         # But BPE models share a vocabulary across training corpora.
         training_vocab=as.factor(ifelse(str_detect(training, "gptbpe"), "gptbpe", as.character(training))),
         training_source=as.factor(str_replace(as.character(training), "-gptbpe", ""))
         ) %>%
  mutate(seed = as.factor(seed)) %>%
  select(-pid, -test_loss) %>%
  distinct(model, training, seed, .keep_all = TRUE)
table(language_model_data$seed)

         0        111        120        922       1111       3602       4301       7245       7877      28066      28068      44862      51272      64924 1581807512 1581807578 1581861474 1581955288 1582126320 1611262494 1611265210 
         4          7          6          5          4          1          1          1          1          1          1          1          1          1          1          1          1          1          1          1          1 

table(model_deltas$seed)

       111        120        607        922       1111       3602       4301       7245       7877      28066      28068      44862      51272      64924 1581807512 1581807578 1586986276 1587139950 1611262494 1611265210 
         9          9          1          9         14          3          3          3          3          3          3          3          3          3          3          3          3          3          3          3 

First join delta-metric data with model auxiliary data.

model_deltas = model_deltas %>%
  merge(language_model_data, by = c("seed", "training", "model"), all=T) %>%
  drop_na()

model_deltas

Also join on the original linear model data, rather than collapsing to delta-metrics. This will support regressions later on that don’t collapse across folds.

Final data preprocessing

# Exclude ordered-neurons from all analyses.
model_deltas <- model_deltas %>%
  filter(model != "ordered-neurons")

# Exclude bad GPT models.
model_deltas <- model_deltas %>%
  filter(model != "gpt2" | !(seed %in% c(1581955288, 1581861474, 1582126320)))

Visualizations

The basics

all_data %>% ggplot(aes(x=corpus)) + geom_bar()

print(all_data %>% group_by(corpus) %>% summarise(n=n()))

all_data %>% 
  ggplot(aes(x=freq, color=corpus)) + geom_density()

all_data %>% 
  ggplot(aes(x=len, color=corpus)) + geom_density()

all_data %>% 
  ggplot(aes(x=surprisal, color=corpus)) + geom_density()

Predictive power and SG

model_deltas %>%
  ggplot(aes(x=sg_score, y=delta_test_mean)) +
    geom_errorbar(aes(ymin=delta_test_mean-delta_test_sem, ymax=delta_test_mean+delta_test_sem)) +
    geom_smooth(method="lm", se=T) +
    geom_point(stat="identity", position="dodge", alpha=1, size=3, aes(color=training_vocab, shape=model)) +
    ylab(metric) +
    xlab("Syntax Generalization Score") +
    ggtitle("Syntactic Generalization vs. Predictive Power") +
    scale_color_manual(values = c("bllip-lg"="#440154FF",
                              "bllip-md"="#39568CFF",
                              "bllip-sm"="#1F968BFF",
                              "bllip-xs"="#73D055FF",
                              "gptbpe"="#888888")) +
    facet_grid(~corpus, scales="free") +
    theme(axis.text=element_text(size=14),
          strip.text.x = element_text(size=14),
          legend.text=element_text(size=14),
          axis.title=element_text(size=18),
          legend.position = "bottom")

#ggsave("./cogsci_images/sg_loglik.png",height=5,width=6)

Regression analyses

We control for effects of perplexity by relating the residuals of a performance ~ PPL regression to SG score.

# Prepare a residualized regression for x1 onto y, controlling for the effects of x2.
d_resid = model_deltas %>%
  drop_na() %>%
  
  group_by(corpus) %>%
    # Residualize delta metric w.r.t PPL for each model--training--seed within
    # training vocabulary
    mutate(resid.delta = resid(lm(delta_test_mean ~ training_vocab:test_ppl))) %>%
    # Residualize SG score w.r.t. PPL for each model--training--seed
    # within training vocabulary
    mutate(resid.sg = resid(lm(sg_score ~ training_vocab:test_ppl))) %>%
  ungroup()


# Now plot residual vs SG
d_resid %>%
  ggplot(aes(x=resid.sg, y=resid.delta)) +
    theme_bw() +
    scale_shape_manual(values = c(21, 24, 22, 23)) +
    geom_smooth(method="lm", se=T, alpha=0.3) +
    geom_point(stat="identity", position="dodge", alpha=1, size=5, aes(shape = model, fill=training_source, color = training_vocab, stroke = 1)) +
    ylab(paste("Residual", metric)) +
    xlab("Residual Syntax Generalization Score") +
    ggtitle("Syntactic Generalization vs. Predictive Power") +
    labs(color="training") + 
    scale_color_manual(values = c("bllip-lg"="#440154FF",
                                  "bllip-md"="#39568CFF",
                                  "bllip-sm"="#1F968BFF",
                                  "bllip-xs"="#73D055FF",
                                  "gptbpe"="#f0941f")) +
  scale_fill_manual(values = c("bllip-lg"="#440154FF",
                                  "bllip-md"="#39568CFF",
                                  "bllip-sm"="#1F968BFF",
                                  "bllip-xs"="#73D055FF",
                                  "gptbpe"="#f0941f"), guide=F) +
    facet_grid(.~corpus, scales="free") +
    theme(axis.text=element_text(size=14),
          strip.text.x = element_text(size=14),
          legend.text=element_text(size=14),
          axis.title=element_text(size=18),
          legend.position = "right")
ggsave("../images/cogsci2020/dll_sg.pdf",height=4.5,width=9, device = cairo_pdf)

do_stepwise_regression = function(cur_corpus) {
  regression_data = model_deltas %>%
    filter(corpus == cur_corpus)
  
  print("----------------------")
  print(cur_corpus)
  
  lm1 = lm(delta_test_mean ~ training_vocab:test_ppl, data = regression_data)
  lm2 = lm(delta_test_mean ~ training_vocab:test_ppl + sg_score, data = regression_data)
  print(anova(lm1, lm2))
  summary(lm2)
}
do_stepwise_regression("bnc-brown")

[1] "----------------------"
[1] "bnc-brown"
Analysis of Variance Table

Model 1: delta_test_mean ~ training_vocab:test_ppl
Model 2: delta_test_mean ~ training_vocab:test_ppl + sg_score
  Res.Df        RSS Df  Sum of Sq     F Pr(>F)  
1     20 0.00010381                             
2     19 0.00007619  1 2.7617e-05 6.887 0.0167 *
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Call:
lm(formula = delta_test_mean ~ training_vocab:test_ppl + sg_score, 
    data = regression_data)

Residuals:
       Min         1Q     Median         3Q        Max 
-0.0030275 -0.0006859 -0.0001488  0.0004360  0.0049661 

Coefficients:
                                  Estimate Std. Error t value Pr(>|t|)    
(Intercept)                      1.603e-02  1.937e-03   8.277 1.01e-07 ***
sg_score                        -9.472e-03  3.609e-03  -2.624  0.01670 *  
training_vocabbllip-lg:test_ppl  3.858e-05  1.904e-05   2.026  0.05701 .  
training_vocabbllip-md:test_ppl -1.215e-05  1.622e-05  -0.749  0.46293    
training_vocabbllip-sm:test_ppl -4.043e-05  1.374e-05  -2.943  0.00835 ** 
training_vocabbllip-xs:test_ppl -4.913e-05  9.091e-06  -5.405 3.25e-05 ***
training_vocabgptbpe:test_ppl   -1.010e-05  4.768e-06  -2.118  0.04761 *  
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.002002 on 19 degrees of freedom
Multiple R-squared:   0.78, Adjusted R-squared:  0.7106 
F-statistic: 11.23 on 6 and 19 DF,  p-value: 2.192e-05

do_stepwise_regression("dundee")

[1] "----------------------"
[1] "dundee"
Analysis of Variance Table

Model 1: delta_test_mean ~ training_vocab:test_ppl
Model 2: delta_test_mean ~ training_vocab:test_ppl + sg_score
  Res.Df        RSS Df  Sum of Sq      F    Pr(>F)    
1     20 4.3511e-05                                   
2     19 1.6136e-05  1 2.7375e-05 32.235 1.792e-05 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Call:
lm(formula = delta_test_mean ~ training_vocab:test_ppl + sg_score, 
    data = regression_data)

Residuals:
       Min         1Q     Median         3Q        Max 
-0.0013727 -0.0004759 -0.0000158  0.0003643  0.0019568 

Coefficients:
                                  Estimate Std. Error t value Pr(>|t|)    
(Intercept)                      9.938e-03  8.915e-04  11.148 8.90e-10 ***
sg_score                        -9.430e-03  1.661e-03  -5.678 1.79e-05 ***
training_vocabbllip-lg:test_ppl  1.598e-06  8.762e-06   0.182  0.85719    
training_vocabbllip-md:test_ppl  1.152e-06  7.465e-06   0.154  0.87895    
training_vocabbllip-sm:test_ppl -1.346e-05  6.322e-06  -2.129  0.04652 *  
training_vocabbllip-xs:test_ppl -1.582e-05  4.184e-06  -3.781  0.00126 ** 
training_vocabgptbpe:test_ppl   -2.323e-06  2.194e-06  -1.059  0.30307    
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.0009215 on 19 degrees of freedom
Multiple R-squared:  0.7306,    Adjusted R-squared:  0.6456 
F-statistic: 8.589 on 6 and 19 DF,  p-value: 0.0001339

do_stepwise_regression("natural-stories")

[1] "----------------------"
[1] "natural-stories"
Analysis of Variance Table

Model 1: delta_test_mean ~ training_vocab:test_ppl
Model 2: delta_test_mean ~ training_vocab:test_ppl + sg_score
  Res.Df        RSS Df  Sum of Sq      F  Pr(>F)  
1     20 2.8774e-05                               
2     19 2.1659e-05  1 7.1149e-06 6.2416 0.02182 *
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Call:
lm(formula = delta_test_mean ~ training_vocab:test_ppl + sg_score, 
    data = regression_data)

Residuals:
       Min         1Q     Median         3Q        Max 
-0.0022008 -0.0005803  0.0002550  0.0006805  0.0013215 

Coefficients:
                                  Estimate Std. Error t value Pr(>|t|)    
(Intercept)                      1.105e-02  1.033e-03  10.695 1.77e-09 ***
sg_score                        -4.808e-03  1.924e-03  -2.498  0.02182 *  
training_vocabbllip-lg:test_ppl -6.151e-05  1.015e-05  -6.059 7.92e-06 ***
training_vocabbllip-md:test_ppl -5.557e-05  8.649e-06  -6.426 3.68e-06 ***
training_vocabbllip-sm:test_ppl -6.020e-05  7.325e-06  -8.219 1.12e-07 ***
training_vocabbllip-xs:test_ppl -3.515e-05  4.847e-06  -7.251 6.99e-07 ***
training_vocabgptbpe:test_ppl   -7.707e-06  2.542e-06  -3.032  0.00686 ** 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 0.001068 on 19 degrees of freedom
Multiple R-squared:  0.8196,    Adjusted R-squared:  0.7626 
F-statistic: 14.38 on 6 and 19 DF,  p-value: 3.641e-06

# The residualized analysis and the stepwise regression analysis
# should yield the same coefficients for the SG score variable.
#
# Below, we compute the slope coefficient for the SG term in the
# residualized analyses.
#
# These coefficients should match those found in the stepwise
# regression for `sg_score` above.
d_resid %>% group_by(corpus) %>%
  group_modify(~tidy(lm(resid.delta ~ training_vocab:test_ppl + resid.sg, data=.))
                 %>% filter(term == "resid.sg")) %>% 
  select(corpus, estimate)

Predictive power and perplexity

model_deltas %>%
  mutate(test_ppl = if_else(test_ppl > 500, 329.9, test_ppl),
         bpe = if_else(training_vocab == "gptbpe", "yes", "no")) %>%
  ggplot(aes(x=test_ppl, y=delta_test_mean, shape = model, ymin=0)) +
    theme_bw() +
    geom_text(aes(x=275, y=0, label = c("//"))) +
    geom_errorbar(aes(ymin=delta_test_mean-delta_test_sem, ymax=delta_test_mean+delta_test_sem, color=training_vocab), alpha=0.4) +
    #geom_smooth(method="lm", se=F) +
    geom_point(stat="identity", position="dodge", alpha=1, size=5, aes(fill=training_source, color = training_vocab, stroke = 1)) +
    ylab("ΔLogLik per token") +
    xlab("Test Perplexity") +
    #coord_cartesian(ylim = c(1, 16)) +
    ggtitle("Test Perplexity vs. Predictive Power") +
    labs(color="training") + 
    scale_color_manual(values = c("bllip-lg"="#440154FF",
                                  "bllip-md"="#39568CFF",
                                  "bllip-sm"="#1F968BFF",
                                  "bllip-xs"="#73D055FF",
                                  "gptbpe"="#f0941f")) +
  scale_fill_manual(values = c("bllip-lg"="#440154FF",
                                  "bllip-md"="#39568CFF",
                                  "bllip-sm"="#1F968BFF",
                                  "bllip-xs"="#73D055FF",
                                  "gptbpe"="#f0941f"), guide=F) +
    scale_shape_manual(values = c(21, 24, 22, 23)) +
    scale_x_continuous(labels=c(0, 50, 100, 150, 200, 250, 500 ,550), breaks=c(0, 50, 100, 150, 200, 250, 300, 350), minor_breaks = NULL) +
    scale_y_continuous(limits = c(0, NA), expand = c(0,0)) +
    facet_wrap(~corpus, scales="free") +
    coord_cartesian(clip="off") +
    theme(axis.text=element_text(size=12),
          strip.text.x = element_text(size=12),
          legend.text=element_text(size=12),
          axis.title=element_text(size=12),
          legend.position = "right")
ggsave("../images/cogsci2020/ppl_loglik.pdf",height=5,width=12, device = cairo_pdf)

Regression: Impact of PPL on Predictive Power

lmd = model_deltas %>%
  mutate(training_vocab=ifelse(str_detect(as.character(training), "gptbpe"),
                               "gptbpe", as.character(training)))
summary(lmer(delta_test_mean ~ training_vocab:test_ppl + (1 | corpus) + (1 | model), data=lmd))

Linear mixed model fit by REML. t-tests use Satterthwaite's method ['lmerModLmerTest']
Formula: delta_test_mean ~ training_vocab:test_ppl + (1 | corpus) + (1 |      model)
   Data: lmd

REML criterion at convergence: -616.7

Scaled residuals: 
    Min      1Q  Median      3Q     Max 
-2.5463 -0.4997 -0.1887  0.4441  2.6177 

Random effects:
 Groups   Name        Variance  Std.Dev.
 model    (Intercept) 2.200e-06 0.001483
 corpus   (Intercept) 6.036e-06 0.002457
 Residual             3.886e-06 0.001971
Number of obs: 78, groups:  model, 4; corpus, 3

Fixed effects:
                                  Estimate Std. Error         df t value Pr(>|t|)   
(Intercept)                      9.411e-03  1.996e-03  4.503e+00   4.716  0.00686 **
training_vocabbllip-lg:test_ppl -7.790e-06  2.092e-05  3.549e+00  -0.372  0.73078   
training_vocabbllip-md:test_ppl -2.372e-05  1.832e-05  3.390e+00  -1.294  0.27666   
training_vocabbllip-sm:test_ppl -3.346e-05  1.506e-05  3.355e+00  -2.222  0.10334   
training_vocabbllip-xs:test_ppl -2.861e-05  1.017e-05  3.269e+00  -2.813  0.06079 . 
training_vocabgptbpe:test_ppl   -7.387e-06  2.984e-06  6.939e+01  -2.475  0.01575 * 
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Correlation of Fixed Effects:
                  (Intr) trnng_vcbbllp-l:_ trnng_vcbbllp-m:_ trnng_vcbbllp-s:_ trnng_vcbbllp-x:_
trnng_vcbbllp-l:_ -0.542                                                                        
trnng_vcbbllp-m:_ -0.554  0.863                                                                 
trnng_vcbbllp-s:_ -0.557  0.868             0.887                                               
trnng_vcbbllp-x:_ -0.563  0.876             0.898             0.902                             
trnng_vcb:_       -0.227  0.249             0.254             0.256             0.258           

Perplexity vs. SG Score

This is a reproduction of Figure 2 from Hu et al.

model_deltas %>%
  mutate(test_ppl = if_else(test_ppl > 500, 329.9, test_ppl)) %>%
  mutate(train_size = log(train_size)) %>%
  mutate(bpe = if_else(training_vocab == "gptbpe", "yes", "no"),
         bpe = as.factor(bpe)) %>%
  ggplot(aes(x=test_ppl, y=sg_score)) +
    theme_bw() +
    geom_hline(yintercept = 0.28, linetype = "dashed", color="gray") +
    geom_text(aes(x=240, y=0.3), label="random", color="gray") +
    geom_point(stat="identity", position="dodge", alpha=0.3, size=4, aes(shape = model, fill=training_source, color = training_vocab, stroke = 1)) +
    geom_text(aes(x=275, y=0, label = c("//"))) +
    ylab("SG Score") +
    xlab("Test Perplexity") +
    ggtitle("Test PPL vs. SG Score") +
    labs(color="training") + 
    scale_color_manual(values = c("bllip-lg"="#440154FF",
                                  "bllip-md"="#39568CFF",
                                  "bllip-sm"="#1F968BFF",
                                  "bllip-xs"="#73D055FF",
                                  "gptbpe"="#f0941f")) +
    scale_fill_manual(values = c("bllip-lg"="#440154FF",
                                  "bllip-md"="#39568CFF",
                                  "bllip-sm"="#1F968BFF",
                                  "bllip-xs"="#73D055FF",
                                  "gptbpe"="#f0941f"), guide=F) +
    scale_shape_manual(values = c("5gram"=21, vanilla=22, gpt2=24, rnng=23)) +
    scale_x_continuous(labels=c(0, 50, 100, 150, 200, 250, 500 ,550), breaks=c(0, 50, 100, 150, 200, 250, 300, 350), minor_breaks = NULL) +
    scale_y_continuous(limits = c(0, 1), expand = c(0,0)) +
    theme(axis.text=element_text(size=12),
          strip.text.x = element_text(size=12),
          legend.text=element_text(size=8),
          legend.title=element_text(size=8),
          axis.title=element_text(size=14),
          legend.position = "none",
          legend.direction = "horizontal",
          legend.key.width = unit(0.3,"cm"),
          legend.spacing.x = unit(0.1, 'cm'))
ggsave("../images/cogsci2020/ppl_sg.pdf",height=4.5,width=3, device = cairo_pdf)

Smith & Levy reproduction

This redone so that it’s unique for each model

all_data %>%
  ggplot(aes(x=surprisal, color=model)) +
  theme_bw() +
  geom_density() +
  facet_wrap(.~corpus, ncol=1, scales="free", strip.position = "right") +
  coord_cartesian(xlim = c(0, 25)) +
  ggtitle("Distribution of Surprisal") +
ggsave("../images/cogsci2020/surp_corr_marginals.png",height=5,width=4)

k = 1.97

# Fit a GAM for a bootstrap sample.
fit_gam_inner = function(bootstrap_sample, key) {
  # This bootstrap sample may have repeated elements. That causes a problem for
  # mgcv, which internally cross-validates some model parameters -- it may
  # allocate repeated elements to different folds and thus double-dip. We'll
  # prevent this by instead providing the whole (pre-bootstrap) dataset to mgcv,
  # and using `weights` to constrain which elements are seen, and how many
  # times. (Repeated elements of the sample may get a weight of 2 or 3 or N,
  # which is exactly what we want.)
  
  # rsplit$data contains the original entire dataset.
  df = bootstrap_sample$data
  # as.integer.rsplit returns the indices of the examples which are in-sample.
  # convert this to a count vector, with dimension N (total dataset rows)
  weights = tabulate(as.integer(bootstrap_sample), nrow(df))
  
  if (key$corpus == "dundee") {
    # Reading time regression: use features of current and previous word
    m = gam(psychometric ~ s(surprisal, bs = 'cr', k = 20) + s(prev_surp, bs = 'cr', k = 20) +
                           te(freq, len, bs = 'cr') + te(prev_freq, prev_len, bs = 'cr'),
            data = df, weights = weights)
    
    terms_to_predict = c("s(surprisal)", "s(prev_surp)")
  } else {
    # SPRT regression: use features of current and 3 previous words
    m = gam(psychometric ~ s(surprisal, bs = 'cr', k = 20) + s(prev_surp, bs = 'cr', k = 20) +
                           s(prev2_surp, bs = 'cr', k = 20) + s(prev3_surp, bs = 'cr', k = 20) +
                           te(freq, len, bs = 'cr') + te(prev_freq, prev_len, bs = 'cr') +
                           te(prev2_freq, prev2_len, bs = 'cr') + te(prev3_freq, prev3_len, bs = 'cr'),
            data = df, weights = weights)
    
    terms_to_predict = c("s(surprisal)", "s(prev_surp)",
                         "s(prev2_surp)", "s(prev3_surp)")
  }

  # Produce psychometric predictions line using just the relevant context-specific predictors.

  newdata = data.frame(surprisal=seq(0,20,by=0.1),
                       prev_surp=seq(0,20,by=0.1),
                       prev2_surp=seq(0,20,by=0.1),
                       prev3_surp=seq(0,20,by=0.1),
                       freq=0, prev_freq=0, prev2_freq=0, prev3_freq=0,
                       len=0, prev_len=0, prev2_len=0, prev3_len=0)
  
  # Returns a matrix N_samples * N_terms.
  per_term_predictions = predict(m, newdata=newdata, terms=terms_to_predict, type="terms")
  
  # Additive model -- sum across predictor response contributions (matrix columns).
  predictions = rowSums(per_term_predictions)
  
  return(newdata %>% mutate(y=predictions))
}

# Fit a bootstrap-re-estimated GAM for the given model--corpus--training group.
fit_gam = function(df, key, alpha=0.05) {
  # Bootstrap-resample data
  boot_models = df %>% bootstraps(times=50) %>% 
    # Fit a GAM and get predictions for each sample
    mutate(smoothed=map(splits, fit_gam_inner, key=key))
  
  # Extract mean and 5% and 95% percentile y-values for each surprisal value
  result = boot_models %>% 
    unnest(smoothed) %>% 
    select(surprisal, y) %>% 
    group_by(surprisal) %>% 
      summarise(y_lower=quantile(y, alpha / 2), 
                y_upper=quantile(y, 1 - alpha / 2),
                y=mean(y)) %>% 
    ungroup()
  
  return (result)
}

smooths = all_data %>%
  mutate(
    training_vocab=as.factor(ifelse(str_detect(training, "gptbpe"), "gptbpe", as.character(training))),
    training_source=as.factor(str_replace(as.character(training), "-gptbpe", ""))) %>%
  group_by(training_vocab, training_source, model, corpus) %>%
    group_modify(fit_gam) %>%
  ungroup()
write.csv(smooths, "../data/gam_smooths.csv")

Plot the GAM model fits

ymin = -40
ymax = 100
xmin = 0
xmax = 20

get_d_points = function(df, model, training, corpus){
  x = density(df$surprisal)$x
  y = density(df$surprisal)$y
  return(data.frame(model, training, corpus, x, y))
}

# Get the density points
density_data = all_data %>%
  mutate(model = recode(model, vanilla="lstm")) %>%
  group_by(model, training, corpus) %>%
    do({get_d_points(., unique(.$model), unique(.$training), unique(.$corpus))}) %>%
  ungroup() %>%
  mutate(training_vocab=as.factor(ifelse(str_detect(training, "gptbpe"), "gptbpe", as.character(training))),
         training_source=as.factor(str_replace(as.character(training), "-gptbpe", "")),
         bpe=training_vocab == "gptbpe",) %>% 
  filter(x>0, x<20)

smooths = read.csv("../data/gam_smooths.sl2013.all.csv")

cannot open file '../data/gam_smooths.sl2013.all.csv': No such file or directoryError in file(file, "rt") : cannot open the connection